From da8fa4ab98a33bce58ddb0cf9c49202fc423457c Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Mon, 1 Feb 2016 21:44:46 +0300 Subject: [PATCH 1/4] :bug: Don't scope decimals as illegal --- grammars/javascript.cson | 6 ++---- spec/javascript-spec.coffee | 9 --------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index b37c984d..7e44ff08 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -1211,15 +1211,13 @@ 'patterns': [ { 'comment': '`prop` in `obj.prop`, `func().prop`' - 'match': '(\\.)\\s*(?:(\\b\\d\\w*)|([A-Z][A-Z0-9_$]*\\b\\$*)|([\\w$]*))' + 'match': '(\\.)\\s*(?:([A-Z][A-Z0-9_$]*\\b\\$*)|(\\$*[a-zA-Z_$][\\w$]*))' 'captures': '1': 'name': 'meta.delimiter.property.period.js' '2': - 'name': 'invalid.illegal.js' - '3': 'name': 'constant.other.property.js' - '4': + '3': 'name': 'variable.other.property.js' } ] diff --git a/spec/javascript-spec.coffee b/spec/javascript-spec.coffee index 2e7d26e3..633bb68f 100644 --- a/spec/javascript-spec.coffee +++ b/spec/javascript-spec.coffee @@ -1051,10 +1051,6 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('123a.prop') expect(tokens[0]).toEqual value: '123a', scopes: ['source.js', 'invalid.illegal.js'] - it "doesn't confuse illegal objects with numbers", -> - {tokens} = grammar.tokenizeLine('123.') - expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.js'] - describe "function calls", -> it "tokenizes function calls", -> {tokens} = grammar.tokenizeLine('functionCall()') @@ -1220,11 +1216,6 @@ describe "Javascript grammar", -> expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'variable.other.property.js'] - {tokens} = grammar.tokenizeLine('a.123illegal') - expect(tokens[0]).toEqual value: 'a', scopes: ['source.js'] - expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] - expect(tokens[2]).toEqual value: '123illegal', scopes: ['source.js', 'invalid.illegal.js'] - it "tokenizes constant properties", -> {tokens} = grammar.tokenizeLine('obj.MY_CONSTANT') expect(tokens[0]).toEqual value: 'obj', scopes: ['source.js', 'variable.other.object.js'] From 58c6a2f9185386cc7acd2ef630aa71621600deb2 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Mon, 1 Feb 2016 21:48:56 +0300 Subject: [PATCH 2/4] Tokenize decimal marks --- grammars/javascript.cson | 31 ++++++++++++++-- spec/javascript-spec.coffee | 72 ++++++++++++++++++++----------------- 2 files changed, 69 insertions(+), 34 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 7e44ff08..bf7c2848 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -918,8 +918,35 @@ 'numbers': 'patterns': [ { - 'match': '\\b(? {tokens} = grammar.tokenizeLine('[1, /test/]') expect(tokens[0]).toEqual value: '[', scopes: ['source.js', 'meta.brace.square.js'] - expect(tokens[1]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.js'] + expect(tokens[1]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] expect(tokens[2]).toEqual value: ',', scopes: ['source.js', 'meta.delimiter.object.comma.js'] expect(tokens[3]).toEqual value: ' ', scopes: ['source.js', 'string.regexp.js'] expect(tokens[4]).toEqual value: '/', scopes: ['source.js', 'string.regexp.js', 'punctuation.definition.string.begin.js'] @@ -179,34 +179,54 @@ describe "Javascript grammar", -> describe "numbers", -> it "tokenizes hexadecimals", -> {tokens} = grammar.tokenizeLine('0x1D306') - expect(tokens[0]).toEqual value: '0x1D306', scopes: ['source.js', 'constant.numeric.js'] + expect(tokens[0]).toEqual value: '0x1D306', scopes: ['source.js', 'constant.numeric.hex.js'] {tokens} = grammar.tokenizeLine('0X1D306') - expect(tokens[0]).toEqual value: '0X1D306', scopes: ['source.js', 'constant.numeric.js'] + expect(tokens[0]).toEqual value: '0X1D306', scopes: ['source.js', 'constant.numeric.hex.js'] it "tokenizes binary literals", -> {tokens} = grammar.tokenizeLine('0b011101110111010001100110') - expect(tokens[0]).toEqual value: '0b011101110111010001100110', scopes: ['source.js', 'constant.numeric.js'] + expect(tokens[0]).toEqual value: '0b011101110111010001100110', scopes: ['source.js', 'constant.numeric.binary.js'] {tokens} = grammar.tokenizeLine('0B011101110111010001100110') - expect(tokens[0]).toEqual value: '0B011101110111010001100110', scopes: ['source.js', 'constant.numeric.js'] + expect(tokens[0]).toEqual value: '0B011101110111010001100110', scopes: ['source.js', 'constant.numeric.binary.js'] it "tokenizes octal literals", -> {tokens} = grammar.tokenizeLine('0o1411') - expect(tokens[0]).toEqual value: '0o1411', scopes: ['source.js', 'constant.numeric.js'] + expect(tokens[0]).toEqual value: '0o1411', scopes: ['source.js', 'constant.numeric.octal.js'] {tokens} = grammar.tokenizeLine('0O1411') - expect(tokens[0]).toEqual value: '0O1411', scopes: ['source.js', 'constant.numeric.js'] + expect(tokens[0]).toEqual value: '0O1411', scopes: ['source.js', 'constant.numeric.octal.js'] + + {tokens} = grammar.tokenizeLine('0010') + expect(tokens[0]).toEqual value: '0010', scopes: ['source.js', 'constant.numeric.octal.js'] it "tokenizes decimals", -> {tokens} = grammar.tokenizeLine('1234') - expect(tokens[0]).toEqual value: '1234', scopes: ['source.js', 'constant.numeric.js'] + expect(tokens[0]).toEqual value: '1234', scopes: ['source.js', 'constant.numeric.decimal.js'] {tokens} = grammar.tokenizeLine('5e-10') - expect(tokens[0]).toEqual value: '5e-10', scopes: ['source.js', 'constant.numeric.js'] + expect(tokens[0]).toEqual value: '5e-10', scopes: ['source.js', 'constant.numeric.decimal.js'] {tokens} = grammar.tokenizeLine('5E+5') - expect(tokens[0]).toEqual value: '5E+5', scopes: ['source.js', 'constant.numeric.js'] + expect(tokens[0]).toEqual value: '5E+5', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('9.') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js'] + + {tokens} = grammar.tokenizeLine('.9') + expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js'] + expect(tokens[1]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('9.9') + expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js'] + expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] + + {tokens} = grammar.tokenizeLine('.1e-23') + expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js'] + expect(tokens[1]).toEqual value: '1e-23', scopes: ['source.js', 'constant.numeric.decimal.js'] it "does not tokenize numbers that are part of a variable", -> {tokens} = grammar.tokenizeLine('hi$1') @@ -300,9 +320,9 @@ describe "Javascript grammar", -> it "tokenizes the arithmetic operators when separated by newlines", -> for operator in operators lines = grammar.tokenizeLines '1\n' + operator + ' 2' - expect(lines[0][0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.js'] + expect(lines[0][0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] expect(lines[1][0]).toEqual value: operator, scopes: ['source.js', 'keyword.operator.js'] - expect(lines[1][2]).toEqual value: '2', scopes: ['source.js', 'constant.numeric.js'] + expect(lines[1][2]).toEqual value: '2', scopes: ['source.js', 'constant.numeric.decimal.js'] describe "assignment", -> it "tokenizes '=' operator", -> @@ -338,7 +358,7 @@ describe "Javascript grammar", -> expect(tokens[3]).toEqual value: ' ', scopes: ['source.js'] expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] expect(tokens[5]).toEqual value: ' ', scopes: ['source.js'] - expect(tokens[6]).toEqual value: '42', scopes: ['source.js', 'constant.numeric.js'] + expect(tokens[6]).toEqual value: '42', scopes: ['source.js', 'constant.numeric.decimal.js'] expect(tokens[7]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] {tokens} = grammar.tokenizeLine('something = MY_COOL_VAR * 1;') @@ -349,7 +369,7 @@ describe "Javascript grammar", -> expect(tokens[4]).toEqual value: ' ', scopes: ['source.js'] expect(tokens[5]).toEqual value: '*', scopes: ['source.js', 'keyword.operator.js'] expect(tokens[6]).toEqual value: ' ', scopes: ['source.js'] - expect(tokens[7]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.js'] + expect(tokens[7]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] expect(tokens[8]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] it "tokenizes variables declared using `const` as constants", -> @@ -360,7 +380,7 @@ describe "Javascript grammar", -> expect(tokens[3]).toEqual value: ' ', scopes: ['source.js'] expect(tokens[4]).toEqual value: '=', scopes: ['source.js', 'keyword.operator.assignment.js'] expect(tokens[5]).toEqual value: ' ', scopes: ['source.js'] - expect(tokens[6]).toEqual value: '42', scopes: ['source.js', 'constant.numeric.js'] + expect(tokens[6]).toEqual value: '42', scopes: ['source.js', 'constant.numeric.decimal.js'] expect(tokens[7]).toEqual value: ';', scopes: ['source.js', 'punctuation.terminator.statement.js'] lines = grammar.tokenizeLines """ @@ -763,7 +783,7 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('export default 123;') expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] expect(tokens[2]).toEqual value: 'default', scopes: ['source.js', 'meta.export.js', 'variable.language.default.js'] - expect(tokens[4]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.js'] + expect(tokens[4]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.decimal.js'] {tokens} = grammar.tokenizeLine('export default name;') expect(tokens[0]).toEqual value: 'export', scopes: ['source.js', 'meta.export.js', 'keyword.control.js'] @@ -1070,14 +1090,14 @@ describe "Javascript grammar", -> expect(tokens[10]).toEqual value: '{', scopes: ['source.js', 'meta.function-call.js', 'meta.brace.curly.js'] expect(tokens[11]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js'] expect(tokens[12]).toEqual value: ':', scopes: ['source.js', 'meta.function-call.js', 'keyword.operator.js'] - expect(tokens[14]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'constant.numeric.js'] + expect(tokens[14]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'constant.numeric.decimal.js'] expect(tokens[15]).toEqual value: '}', scopes: ['source.js', 'meta.function-call.js', 'meta.brace.curly.js'] expect(tokens[16]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js'] {tokens} = grammar.tokenizeLine('functionCall((123).toString())') expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js'] expect(tokens[2]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'meta.brace.round.js'] - expect(tokens[3]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'constant.numeric.js'] + expect(tokens[3]).toEqual value: '123', scopes: ['source.js', 'meta.function-call.js', 'constant.numeric.decimal.js'] expect(tokens[4]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'meta.brace.round.js'] expect(tokens[9]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js'] @@ -1104,18 +1124,6 @@ describe "Javascript grammar", -> expect(tokens[2]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js'] it "tokenizes illegal arguments", -> - {tokens} = grammar.tokenizeLine('a(1a)') - expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] - expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js'] - expect(tokens[2]).toEqual value: '1a', scopes: ['source.js', 'meta.function-call.js'] - expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js'] - - {tokens} = grammar.tokenizeLine('a(123a)') - expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] - expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js'] - expect(tokens[2]).toEqual value: '123a', scopes: ['source.js', 'meta.function-call.js'] - expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js'] - {tokens} = grammar.tokenizeLine('a(1.prop)') expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js'] @@ -1147,9 +1155,9 @@ describe "Javascript grammar", -> expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.method-call.js', 'meta.delimiter.method.period.js'] expect(tokens[2]).toEqual value: 'b', scopes: ['source.js', 'meta.method-call.js', 'entity.name.function.js'] expect(tokens[3]).toEqual value: '(', scopes: ['source.js', 'meta.method-call.js', 'punctuation.definition.arguments.begin.js'] - expect(tokens[4]).toEqual value: '1', scopes: ['source.js', 'meta.method-call.js', 'constant.numeric.js'] + expect(tokens[4]).toEqual value: '1', scopes: ['source.js', 'meta.method-call.js', 'constant.numeric.decimal.js'] expect(tokens[5]).toEqual value: '+', scopes: ['source.js', 'meta.method-call.js', 'keyword.operator.js'] - expect(tokens[6]).toEqual value: '1', scopes: ['source.js', 'meta.method-call.js', 'constant.numeric.js'] + expect(tokens[6]).toEqual value: '1', scopes: ['source.js', 'meta.method-call.js', 'constant.numeric.decimal.js'] expect(tokens[7]).toEqual value: ')', scopes: ['source.js', 'meta.method-call.js', 'punctuation.definition.arguments.end.js'] {tokens} = grammar.tokenizeLine('a.$abc$()') From a6c28e8a31f61a6b36b02b2bd235e41e9e7c01a0 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Tue, 2 Feb 2016 01:09:50 +0300 Subject: [PATCH 3/4] Tokenize illegal identifiers --- grammars/javascript.cson | 53 +++++++++++++++++++++++-------------- spec/javascript-spec.coffee | 45 ++++++++++++++++++++++++------- 2 files changed, 69 insertions(+), 29 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index bf7c2848..960d26ee 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -858,15 +858,19 @@ { 'include': '#function_call' } - { - 'include': '#properties' - } { 'include': '#object_variable' } { 'include': '#numbers' } + { + 'include': '#properties' + } + { + 'match': '(? {tokens} = grammar.tokenizeLine('9.') expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] - expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] {tokens} = grammar.tokenizeLine('.9') - expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js'] + expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] expect(tokens[1]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] {tokens} = grammar.tokenizeLine('9.9') expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] - expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js'] {tokens} = grammar.tokenizeLine('.1e-23') - expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal-mark.period.js'] + expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] expect(tokens[1]).toEqual value: '1e-23', scopes: ['source.js', 'constant.numeric.decimal.js'] it "does not tokenize numbers that are part of a variable", -> @@ -1050,6 +1050,16 @@ describe "Javascript grammar", -> {tokens} = grammar.tokenizeLine('super') expect(tokens[0]).toEqual value: 'super', scopes: ['source.js', 'variable.language.js'] + it "tokenizes illegal identifiers", -> + {tokens} = grammar.tokenizeLine('0illegal') + expect(tokens[0]).toEqual value: '0illegal', scopes: ['source.js', 'invalid.illegal.identifier.js'] + + {tokens} = grammar.tokenizeLine('123illegal') + expect(tokens[0]).toEqual value: '123illegal', scopes: ['source.js', 'invalid.illegal.identifier.js'] + + {tokens} = grammar.tokenizeLine('123$illegal') + expect(tokens[0]).toEqual value: '123$illegal', scopes: ['source.js', 'invalid.illegal.identifier.js'] + describe "objects", -> it "tokenizes them", -> {tokens} = grammar.tokenizeLine('obj.prop') @@ -1063,13 +1073,13 @@ describe "Javascript grammar", -> it "tokenizes illegal objects", -> {tokens} = grammar.tokenizeLine('1.prop') - expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'invalid.illegal.js'] + expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'invalid.illegal.identifier.js'] {tokens} = grammar.tokenizeLine('123.prop') - expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'invalid.illegal.js'] + expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'invalid.illegal.identifier.js'] {tokens} = grammar.tokenizeLine('123a.prop') - expect(tokens[0]).toEqual value: '123a', scopes: ['source.js', 'invalid.illegal.js'] + expect(tokens[0]).toEqual value: '123a', scopes: ['source.js', 'invalid.illegal.identifier.js'] describe "function calls", -> it "tokenizes function calls", -> @@ -1119,15 +1129,27 @@ describe "Javascript grammar", -> it "tokenizes illegal function calls", -> {tokens} = grammar.tokenizeLine('0illegal()') - expect(tokens[0]).toEqual value: '0illegal', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.js'] + expect(tokens[0]).toEqual value: '0illegal', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.identifier.js'] expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js'] expect(tokens[2]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js'] it "tokenizes illegal arguments", -> + {tokens} = grammar.tokenizeLine('a(1a)') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js'] + expect(tokens[2]).toEqual value: '1a', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.identifier.js'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js'] + + {tokens} = grammar.tokenizeLine('a(123a)') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] + expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js'] + expect(tokens[2]).toEqual value: '123a', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.identifier.js'] + expect(tokens[3]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js'] + {tokens} = grammar.tokenizeLine('a(1.prop)') expect(tokens[0]).toEqual value: 'a', scopes: ['source.js', 'meta.function-call.js', 'entity.name.function.js'] expect(tokens[1]).toEqual value: '(', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.begin.js'] - expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.js'] + expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'meta.function-call.js', 'invalid.illegal.identifier.js'] expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'meta.function-call.js', 'meta.delimiter.property.period.js'] expect(tokens[4]).toEqual value: 'prop', scopes: ['source.js', 'meta.function-call.js', 'variable.other.property.js'] expect(tokens[5]).toEqual value: ')', scopes: ['source.js', 'meta.function-call.js', 'punctuation.definition.arguments.end.js'] @@ -1224,6 +1246,11 @@ describe "Javascript grammar", -> expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] expect(tokens[4]).toEqual value: 'b', scopes: ['source.js', 'variable.other.property.js'] + {tokens} = grammar.tokenizeLine('a.123illegal') + expect(tokens[0]).toEqual value: 'a', scopes: ['source.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js'] + expect(tokens[2]).toEqual value: '123illegal', scopes: ['source.js', 'invalid.illegal.identifier.js'] + it "tokenizes constant properties", -> {tokens} = grammar.tokenizeLine('obj.MY_CONSTANT') expect(tokens[0]).toEqual value: 'obj', scopes: ['source.js', 'variable.other.object.js'] From 9cf9a3970369b247878e98897ce1549b94769f38 Mon Sep 17 00:00:00 2001 From: Maxim Sokolov Date: Tue, 2 Feb 2016 02:17:50 +0300 Subject: [PATCH 4/4] :bug: Tokenize '1.E3' as number --- grammars/javascript.cson | 9 ++++++--- spec/javascript-spec.coffee | 5 +++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/grammars/javascript.cson b/grammars/javascript.cson index 960d26ee..378b6392 100644 --- a/grammars/javascript.cson +++ b/grammars/javascript.cson @@ -859,10 +859,10 @@ 'include': '#function_call' } { - 'include': '#object_variable' + 'include': '#numbers' } { - 'include': '#numbers' + 'include': '#object_variable' } { 'include': '#properties' @@ -937,12 +937,13 @@ 'match': '''(?x) (? expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] expect(tokens[1]).toEqual value: '1e-23', scopes: ['source.js', 'constant.numeric.decimal.js'] + {tokens} = grammar.tokenizeLine('1.E3') + expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js'] + expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js'] + expect(tokens[2]).toEqual value: 'E3', scopes: ['source.js', 'constant.numeric.decimal.js'] + it "does not tokenize numbers that are part of a variable", -> {tokens} = grammar.tokenizeLine('hi$1') expect(tokens[0]).toEqual value: 'hi$1', scopes: ['source.js']