Skip to content
This repository was archived by the owner on Dec 15, 2022. It is now read-only.

Don't scope decimals as illegal; tokenize decimal marks, illegal identifiers #319

Merged
winstliu merged 4 commits into from
Feb 2, 2016
Merged

Don't scope decimals as illegal; tokenize decimal marks, illegal identifiers #319

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
da8fa4a
:bug: Don't scope decimals as illegal
MaximSokolov Feb 1, 2016
58c6a2f
Tokenize decimal marks
MaximSokolov Feb 1, 2016
a6c28e8
Tokenize illegal identifiers
MaximSokolov Feb 1, 2016
9cf9a39
:bug: Tokenize '1.E3' as number
MaximSokolov Feb 1, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 49 additions & 8 deletions grammars/javascript.cson
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -859,13 +859,17 @@
'include': '#function_call'
}
{
'include': '#properties'
'include': '#numbers'
}
{
'include': '#object_variable'
}
{
'include': '#numbers'
'include': '#properties'
}
{
'match': '(?<!\\$)\\b[0-9]+[\\w$]*'
'name': 'invalid.illegal.identifier.js'
}
{
'match': '\\;'
Expand Down Expand Up @@ -918,8 +922,45 @@
'numbers':
'patterns': [
{
'match': '\\b(?<!\\$|_)((0(x|X)[0-9a-fA-F]+)|(0(b|B)[01]+)|(0(o|O)[0-7]+)|([0-9]+(\\.[0-9]+)?)((e|E)[+-]?[0-9]+)?)\\b'
'name': 'constant.numeric.js'
'match': '\\b(?<!\\$)0(x|X)[0-9a-fA-F]+\\b(?!\\$)'
'name': 'constant.numeric.hex.js'
}
{
'match': '\\b(?<!\\$)0(b|B)[01]+\\b(?!\\$)'
'name': 'constant.numeric.binary.js'
}
{
'match': '\\b(?<!\\$)0(o|O)?[0-7]+\\b(?!\\$)'
'name': 'constant.numeric.octal.js'
}
{
'match': '''(?x)
(?<!\\$)(?:
(?:\\b[0-9]+(\\.)[0-9]+[eE][+-]?[0-9]+\\b)| # 1.1E+3

@winstliu winstliu Feb 1, 2016

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

1.E3 is also possible...it doesn't look like you're catching that right now

(?:\\b[0-9]+(\\.)[eE][+-]?[0-9]+\\b)| # 1.E+3
(?:\\B(\\.)[0-9]+[eE][+-]?[0-9]+\\b)| # .1E+3
(?:\\b[0-9]+[eE][+-]?[0-9]+\\b)| # 1E+3
(?:\\b[0-9]+(\\.)[0-9]+\\b)| # 1.1
(?:\\b[0-9]+(\\.)\\B)| # 1.
(?:\\B(\\.)[0-9]+\\b)| # .1
(?:\\b[0-9]+\\b(?!\\.)) # 1
)(?!\\$)
'''
'captures':
'0':
'name': 'constant.numeric.decimal.js'
'1':
'name': 'meta.delimiter.decimal.period.js'
'2':
'name': 'meta.delimiter.decimal.period.js'
'3':
'name': 'meta.delimiter.decimal.period.js'
'4':
'name': 'meta.delimiter.decimal.period.js'
'5':
'name': 'meta.delimiter.decimal.period.js'
'6':
'name': 'meta.delimiter.decimal.period.js'
}
]
'operators':
Expand Down Expand Up @@ -1178,7 +1219,7 @@
'begin': '(?:(\\d\\w*)|([a-zA-Z_$][\\w$]*))\\s*(\\()'
'beginCaptures':
'1':
'name': 'invalid.illegal.js'
'name': 'invalid.illegal.identifier.js'
'2':
'name': 'entity.name.function.js'
'3':
Expand All @@ -1204,19 +1245,19 @@
'1':
'name': 'variable.other.object.js'
'2':
'name': 'invalid.illegal.js'
'name': 'invalid.illegal.identifier.js'
}
]
'properties':
'patterns': [
{
'comment': '`prop` in `obj.prop`, `func().prop`'
'match': '(\\.)\\s*(?:(\\b\\d\\w*)|([A-Z][A-Z0-9_$]*\\b\\$*)|([\\w$]*))'
'match': '(\\.)\\s*(?:([0-9]+[\\w$]*)|([A-Z][A-Z0-9_$]*\\b\\$*)|(\\$*[a-zA-Z_$][\\w$]*))'
'captures':
'1':
'name': 'meta.delimiter.property.period.js'
'2':
'name': 'invalid.illegal.js'
'name': 'invalid.illegal.identifier.js'
'3':
'name': 'constant.other.property.js'
'4':
Expand Down
95 changes: 63 additions & 32 deletions spec/javascript-spec.coffee
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ describe "Javascript grammar", ->

{tokens} = grammar.tokenizeLine('[1, /test/]')
expect(tokens[0]).toEqual value: '[', scopes: ['source.js', 'meta.brace.square.js']
expect(tokens[1]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[1]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[2]).toEqual value: ',', scopes: ['source.js', 'meta.delimiter.object.comma.js']
expect(tokens[3]).toEqual value: ' ', scopes: ['source.js', 'string.regexp.js']
expect(tokens[4]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js']
Expand Down Expand Up @@ -179,34 +179,59 @@ describe "Javascript grammar", ->
describe "numbers", ->
it "tokenizes hexadecimals", ->
{tokens} = grammar.tokenizeLine('0x1D306')
expect(tokens[0]).toEqual value: '0x1D306', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[0]).toEqual value: '0x1D306', scopes: ['source.js', 'constant.numeric.hex.js']

{tokens} = grammar.tokenizeLine('0X1D306')
expect(tokens[0]).toEqual value: '0X1D306', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[0]).toEqual value: '0X1D306', scopes: ['source.js', 'constant.numeric.hex.js']

it "tokenizes binary literals", ->
{tokens} = grammar.tokenizeLine('0b011101110111010001100110')
expect(tokens[0]).toEqual value: '0b011101110111010001100110', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[0]).toEqual value: '0b011101110111010001100110', scopes: ['source.js', 'constant.numeric.binary.js']

{tokens} = grammar.tokenizeLine('0B011101110111010001100110')
expect(tokens[0]).toEqual value: '0B011101110111010001100110', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[0]).toEqual value: '0B011101110111010001100110', scopes: ['source.js', 'constant.numeric.binary.js']

it "tokenizes octal literals", ->
{tokens} = grammar.tokenizeLine('0o1411')
expect(tokens[0]).toEqual value: '0o1411', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[0]).toEqual value: '0o1411', scopes: ['source.js', 'constant.numeric.octal.js']

{tokens} = grammar.tokenizeLine('0O1411')
expect(tokens[0]).toEqual value: '0O1411', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[0]).toEqual value: '0O1411', scopes: ['source.js', 'constant.numeric.octal.js']

{tokens} = grammar.tokenizeLine('0010')
expect(tokens[0]).toEqual value: '0010', scopes: ['source.js', 'constant.numeric.octal.js']

it "tokenizes decimals", ->
{tokens} = grammar.tokenizeLine('1234')
expect(tokens[0]).toEqual value: '1234', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[0]).toEqual value: '1234', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('5e-10')
expect(tokens[0]).toEqual value: '5e-10', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[0]).toEqual value: '5e-10', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('5E+5')
expect(tokens[0]).toEqual value: '5E+5', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[0]).toEqual value: '5E+5', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('9.')
expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']

{tokens} = grammar.tokenizeLine('.9')
expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
expect(tokens[1]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('9.9')
expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('.1e-23')
expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
expect(tokens[1]).toEqual value: '1e-23', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('1.E3')
expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
expect(tokens[2]).toEqual value: 'E3', scopes: ['source.js', 'constant.numeric.decimal.js']

it "does not tokenize numbers that are part of a variable", ->
{tokens} = grammar.tokenizeLine('hi$1')
Expand Down Expand Up @@ -300,9 +325,9 @@ describe "Javascript grammar", ->
it "tokenizes the arithmetic operators when separated by newlines", ->
for operator in operators
lines = grammar.tokenizeLines '1\n' + operator + ' 2'
expect(lines[0][0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.js']
expect(lines[0][0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(lines[1][0]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.js']
expect(lines[1][2]).toEqual value: '2', scopes: ['source.js', 'constant.numeric.js']
expect(lines[1][2]).toEqual value: '2', scopes: ['source.js', 'constant.numeric.decimal.js']

describe "assignment", ->
it "tokenizes '=' operator", ->
Expand Down Expand Up @@ -338,7 +363,7 @@ describe "Javascript grammar", ->
expect(tokens[3]).toEqual value: ' ', scopes: ['source.js']
expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js']
expect(tokens[5]).toEqual value: ' ', scopes: ['source.js']
expect(tokens[6]).toEqual value: '42', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[6]).toEqual value: '42', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[7]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js']

{tokens} = grammar.tokenizeLine('something = MY_COOL_VAR * 1;')
Expand All @@ -349,7 +374,7 @@ describe "Javascript grammar", ->
expect(tokens[4]).toEqual value: ' ', scopes: ['source.js']
expect(tokens[5]).toEqual value: '*', scopes: ['source.js', 'keyword.operator.js']
expect(tokens[6]).toEqual value: ' ', scopes: ['source.js']
expect(tokens[7]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[7]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[8]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js']

it "tokenizes variables declared using `const` as constants", ->
Expand All @@ -360,7 +385,7 @@ describe "Javascript grammar", ->
expect(tokens[3]).toEqual value: ' ', scopes: ['source.js']
expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js']
expect(tokens[5]).toEqual value: ' ', scopes: ['source.js']
expect(tokens[6]).toEqual value: '42', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[6]).toEqual value: '42', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[7]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js']

lines = grammar.tokenizeLines """
Expand Down Expand Up @@ -763,7 +788,7 @@ describe "Javascript grammar", ->
{tokens} = grammar.tokenizeLine('export default 123;')
expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js']
expect(tokens[2]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js']
expect(tokens[4]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[4]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('export default name;')
expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js']
Expand Down Expand Up @@ -1030,6 +1055,16 @@ describe "Javascript grammar", ->
{tokens} = grammar.tokenizeLine('super')
expect(tokens[0]).toEqual value: 'super', scopes: ['source.js', 'variable.language.js']

it "tokenizes illegal identifiers", ->
{tokens} = grammar.tokenizeLine('0illegal')
expect(tokens[0]).toEqual value: '0illegal', scopes: ['source.js', 'invalid.illegal.identifier.js']

{tokens} = grammar.tokenizeLine('123illegal')
expect(tokens[0]).toEqual value: '123illegal', scopes: ['source.js', 'invalid.illegal.identifier.js']

{tokens} = grammar.tokenizeLine('123$illegal')
expect(tokens[0]).toEqual value: '123$illegal', scopes: ['source.js', 'invalid.illegal.identifier.js']

describe "objects", ->
it "tokenizes them", ->
{tokens} = grammar.tokenizeLine('obj.prop')
Expand All @@ -1043,17 +1078,13 @@ describe "Javascript grammar", ->

it "tokenizes illegal objects", ->
{tokens} = grammar.tokenizeLine('1.prop')
expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'invalid.illegal.js']
expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'invalid.illegal.identifier.js']

{tokens} = grammar.tokenizeLine('123.prop')
expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'invalid.illegal.js']
expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'invalid.illegal.identifier.js']

{tokens} = grammar.tokenizeLine('123a.prop')
expect(tokens[0]).toEqual value: '123a', scopes: ['source.js', 'invalid.illegal.js']

it "doesn't confuse illegal objects with numbers", ->
{tokens} = grammar.tokenizeLine('123.')
expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.js']
expect(tokens[0]).toEqual value: '123a', scopes: ['source.js', 'invalid.illegal.identifier.js']

describe "function calls", ->
it "tokenizes function calls", ->
Expand All @@ -1074,14 +1105,14 @@ describe "Javascript grammar", ->
expect(tokens[10]).toEqual value: '{', scopes: ['source.js', 'meta.function-call.js', 'meta.brace.curly.js']
expect(tokens[11]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js']
expect(tokens[12]).toEqual value: ':', scopes: ['source.js', 'meta.function-call.js', 'keyword.operator.js']
expect(tokens[14]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'constant.numeric.js']
expect(tokens[14]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'constant.numeric.decimal.js']
expect(tokens[15]).toEqual value: '}', scopes: ['source.js', 'meta.function-call.js', 'meta.brace.curly.js']
expect(tokens[16]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']

{tokens} = grammar.tokenizeLine('functionCall((123).toString())')
expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js']
expect(tokens[2]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.brace.round.js']
expect(tokens[3]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'constant.numeric.js']
expect(tokens[3]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'constant.numeric.decimal.js']
expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.brace.round.js']
expect(tokens[9]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']

Expand All @@ -1103,27 +1134,27 @@ describe "Javascript grammar", ->

it "tokenizes illegal function calls", ->
{tokens} = grammar.tokenizeLine('0illegal()')
expect(tokens[0]).toEqual value: '0illegal', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.js']
expect(tokens[0]).toEqual value: '0illegal', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.identifier.js']
expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js']
expect(tokens[2]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']

it "tokenizes illegal arguments", ->
{tokens} = grammar.tokenizeLine('a(1a)')
expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']
expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js']
expect(tokens[2]).toEqual value: '1a', scopes: ['source.js', 'meta.function-call.js']
expect(tokens[2]).toEqual value: '1a', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.identifier.js']
expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']

{tokens} = grammar.tokenizeLine('a(123a)')
expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']
expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js']
expect(tokens[2]).toEqual value: '123a', scopes: ['source.js', 'meta.function-call.js']
expect(tokens[2]).toEqual value: '123a', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.identifier.js']
expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']

{tokens} = grammar.tokenizeLine('a(1.prop)')
expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js']
expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js']
expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.js']
expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.identifier.js']
expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'meta.function-call.js', 'meta.delimiter.property.period.js']
expect(tokens[4]).toEqual value: 'prop', scopes: ['source.js', 'meta.function-call.js', 'variable.other.property.js']
expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js']
Expand Down Expand Up @@ -1151,9 +1182,9 @@ describe "Javascript grammar", ->
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js']
expect(tokens[2]).toEqual value: 'b', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js']
expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'punctuation.definition.arguments.begin.js']
expect(tokens[4]).toEqual value: '1', scopes: ['source.js', 'meta.method-call.js', 'constant.numeric.js']
expect(tokens[4]).toEqual value: '1', scopes: ['source.js', 'meta.method-call.js', 'constant.numeric.decimal.js']
expect(tokens[5]).toEqual value: '+', scopes: ['source.js', 'meta.method-call.js', 'keyword.operator.js']
expect(tokens[6]).toEqual value: '1', scopes: ['source.js', 'meta.method-call.js', 'constant.numeric.js']
expect(tokens[6]).toEqual value: '1', scopes: ['source.js', 'meta.method-call.js', 'constant.numeric.decimal.js']
expect(tokens[7]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'punctuation.definition.arguments.end.js']

{tokens} = grammar.tokenizeLine('a.$abc$()')
Expand Down Expand Up @@ -1223,7 +1254,7 @@ describe "Javascript grammar", ->
{tokens} = grammar.tokenizeLine('a.123illegal')
expect(tokens[0]).toEqual value: 'a', scopes: ['source.js']
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js']
expect(tokens[2]).toEqual value: '123illegal', scopes: ['source.js', 'invalid.illegal.js']
expect(tokens[2]).toEqual value: '123illegal', scopes: ['source.js', 'invalid.illegal.identifier.js']

it "tokenizes constant properties", ->
{tokens} = grammar.tokenizeLine('obj.MY_CONSTANT')
Expand Down