Skip to content
This repository was archived by the owner on Dec 15, 2022. It is now read-only.

Tokenize function and parameter annotations #228

Merged
winstliu merged 6 commits into from
Jan 25, 2018
Merged

Tokenize function and parameter annotations #228

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
292a7fc
Add support for Python3 parameter annotations
Aug 4, 2015
3389faf
Add function annotations
berdario Sep 19, 2015
40d397b
Apply some fixes
berdario Oct 17, 2015
d5482d3
Tokenize function and parameter annotations
Jan 22, 2018
b71c5b2
Merge branch 'master' into wl-function-annotations
Jan 23, 2018
ecaf564
Remove highlighting of missing parameters
Jan 23, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 43 additions & 17 deletions grammars/python.cson
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,14 +242,10 @@
'name': 'storage.modifier.async.python'
'2':
'name': 'storage.type.function.python'
'end': '(\\))\\s*(?:(\\:)|(.*$\\n?))'
'end': ':'
'endCaptures':
'1':
'name': 'punctuation.definition.parameters.end.python'
'2':
'name': 'punctuation.section.function.begin.python'
'3':
'name': 'invalid.illegal.missing-section-begin.python'
'0':
'name': 'punctuation.definition.function.begin.python'
'name': 'meta.function.python'
'patterns': [
{
Expand All @@ -263,34 +259,64 @@
]
}
{
'begin': '(\\()'
'begin': '\\('
'beginCaptures':
'1':
'0':
'name': 'punctuation.definition.parameters.begin.python'
'end': '\\)'
'endCaptures':
'0':
'name': 'punctuation.definition.parameters.end.python'
'contentName': 'meta.function.parameters.python'
'end': '(?=\\)\\s*\\:)'
'patterns': [
{
'include': '#line_comments'
}
{
'include': '#keyword_arguments'
# param = 3
# param: int = 3
'begin': '\\b([a-zA-Z_][\\w_]*)\\s*(?:(:)\\s*([a-zA-Z_][\\w_]*))?\\s*(=)\\s*'
'beginCaptures':
'1':
'name': 'variable.parameter.function.python'
'2':
'name': 'punctuation.separator.python'
'3':
'name': 'storage.type.python'
'4':
'name': 'keyword.operator.assignment.python'
'end': '(?!\\G)'
'patterns': [
{
'include': '$self'
}
]
}
{
# param
# param: int
'match': '\\b([a-zA-Z_][\\w_]*)\\s*(?:(:)\\s*([a-zA-Z_][\\w_]*))?'
'captures':
'1':
'name': 'variable.parameter.function.python'
'2':
'name': 'punctuation.separator.parameters.python'
'match': '\\b([a-zA-Z_][a-zA-Z_0-9]*)\\s*(?:(,)|(?=[\\n\\)]))'
'name': 'punctuation.separator.python'
'3':
'name': 'storage.type.python'
}
{
'match': ','
'name': 'punctuation.separator.parameters.python'
}
]
}
{
# No match, not at the end of the line, and no opening parentheses
'begin': '(?!\\G)(?!\\s*$)(?!.*\\()'
'end': '$'
'name': 'invalid.illegal.missing-parameters.python'
'match': '(->)\\s*([A-Za-z_][\\w_]*)(?=\\s*:)'
'captures':
'1':
'name': 'keyword.operator.function-annotation.python'
'2':
'name': 'storage.type.python'
}
]
}
Expand Down
36 changes: 21 additions & 15 deletions spec/python-spec.coffee
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -634,19 +634,6 @@ describe "Python grammar", ->
expect(tokens[2]).toEqual value: 'def', scopes: ['source.python', 'meta.function.python', 'storage.type.function.python']
expect(tokens[4]).toEqual value: 'test', scopes: ['source.python', 'meta.function.python', 'entity.name.function.python']

it "tokenizes functions that are missing parameters", ->
{tokens} = grammar.tokenizeLine 'def test # whoops'

expect(tokens[0]).toEqual value: 'def', scopes: ['source.python', 'meta.function.python', 'storage.type.function.python']
expect(tokens[2]).toEqual value: 'test', scopes: ['source.python', 'meta.function.python', 'entity.name.function.python']
expect(tokens[3]).toEqual value: ' # whoops', scopes: ['source.python', 'meta.function.python', 'invalid.illegal.missing-parameters.python']

{tokens} = grammar.tokenizeLine 'def test:'

expect(tokens[0]).toEqual value: 'def', scopes: ['source.python', 'meta.function.python', 'storage.type.function.python']
expect(tokens[2]).toEqual value: 'test', scopes: ['source.python', 'meta.function.python', 'entity.name.function.python']
expect(tokens[3]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'invalid.illegal.missing-parameters.python']

it "tokenizes comments inside function parameters", ->
{tokens} = grammar.tokenizeLine('def test(arg, # comment')

Expand Down Expand Up @@ -677,7 +664,27 @@ describe "Python grammar", ->
expect(tokens[2][5]).toEqual value: ' comment', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'comment.line.number-sign.python']
expect(tokens[3][1]).toEqual value: 'config', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'variable.parameter.function.python']
expect(tokens[4][0]).toEqual value: ')', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.parameters.end.python']
expect(tokens[4][1]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'punctuation.section.function.begin.python']
expect(tokens[4][1]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.function.begin.python']

it "tokenizes a function definition with annotations", ->
{tokens} = grammar.tokenizeLine 'def f(a: None, b: int = 3) -> int:'

expect(tokens[0]).toEqual value: 'def', scopes: ['source.python', 'meta.function.python', 'storage.type.function.python']
expect(tokens[2]).toEqual value: 'f', scopes: ['source.python', 'meta.function.python', 'entity.name.function.python']
expect(tokens[3]).toEqual value: '(', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.parameters.begin.python']
expect(tokens[4]).toEqual value: 'a', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'variable.parameter.function.python']
expect(tokens[5]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'punctuation.separator.python']
expect(tokens[7]).toEqual value: 'None', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'storage.type.python']
expect(tokens[8]).toEqual value: ',', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'punctuation.separator.parameters.python']
expect(tokens[10]).toEqual value: 'b', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'variable.parameter.function.python']
expect(tokens[11]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'punctuation.separator.python']
expect(tokens[13]).toEqual value: 'int', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'storage.type.python']
expect(tokens[15]).toEqual value: '=', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'keyword.operator.assignment.python']
expect(tokens[17]).toEqual value: '3', scopes: ['source.python', 'meta.function.python', 'meta.function.parameters.python', 'constant.numeric.integer.decimal.python']
expect(tokens[18]).toEqual value: ')', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.parameters.end.python']
expect(tokens[20]).toEqual value: '->', scopes: ['source.python', 'meta.function.python', 'keyword.operator.function-annotation.python']
expect(tokens[22]).toEqual value: 'int', scopes: ['source.python', 'meta.function.python', 'storage.type.python']
expect(tokens[23]).toEqual value: ':', scopes: ['source.python', 'meta.function.python', 'punctuation.definition.function.begin.python']

it "tokenizes complex function calls", ->
{tokens} = grammar.tokenizeLine "torch.nn.BCELoss()(Variable(bayes_optimal_prob, 1, requires_grad=False), Yvar).data[0]"
Expand Down Expand Up @@ -758,7 +765,6 @@ describe "Python grammar", ->
expect(tokens[9][0]).toEqual value: delim, scopes: ['source.python', scope, 'punctuation.definition.string.end.python']

it "tokenizes SQL inline highlighting on single line with a CTE", ->

{tokens} = grammar.tokenizeLine('\'WITH example_cte AS (SELECT bar FROM foo) SELECT COUNT(*) FROM example_cte\'')

expect(tokens[0]).toEqual value: '\'', scopes: ['source.python', 'string.quoted.single.single-line.python', 'punctuation.definition.string.begin.python']
Expand Down