diff --git a/package.json b/package.json index 6e92a74c2168..c1cab9c60df7 100644 --- a/package.json +++ b/package.json @@ -1594,6 +1594,7 @@ "tree-kill": "^1.1.0", "typescript-char": "^0.0.0", "uint64be": "^1.0.1", + "unicode": "^10.0.0", "untildify": "^3.0.2", "vscode-debugadapter": "^1.0.1", "vscode-debugprotocol": "^1.0.1", diff --git a/src/client/extension.ts b/src/client/extension.ts index 703ce90a481e..489543063bfe 100644 --- a/src/client/extension.ts +++ b/src/client/extension.ts @@ -57,6 +57,7 @@ import { StopWatch } from './telemetry/stopWatch'; import { registerTypes as commonRegisterTerminalTypes } from './terminals/serviceRegistry'; import { ICodeExecutionManager } from './terminals/types'; import { BlockFormatProviders } from './typeFormatters/blockFormatProvider'; +import { OnEnterFormatter } from './typeFormatters/onEnterFormatter'; import { TEST_OUTPUT_CHANNEL } from './unittests/common/constants'; import * as tests from './unittests/main'; import { registerTypes as unitTestsRegisterTypes } from './unittests/serviceRegistry'; @@ -190,6 +191,8 @@ export async function activate(context: vscode.ExtensionContext) { context.subscriptions.push(new WorkspaceSymbols(serviceContainer)); context.subscriptions.push(vscode.languages.registerOnTypeFormattingEditProvider(PYTHON, new BlockFormatProviders(), ':')); + context.subscriptions.push(vscode.languages.registerOnTypeFormattingEditProvider(PYTHON, new OnEnterFormatter(), '\n')); + // In case we have CR LF const triggerCharacters: string[] = os.EOL.split(''); triggerCharacters.shift(); diff --git a/src/client/formatters/lineFormatter.ts b/src/client/formatters/lineFormatter.ts new file mode 100644 index 000000000000..74286590dc5e --- /dev/null +++ b/src/client/formatters/lineFormatter.ts @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// tslint:disable-next-line:import-name +import Char from 'typescript-char'; +import { BraceCounter } from '../language/braceCounter'; +import { TextBuilder } from '../language/textBuilder'; +import { Tokenizer } from '../language/tokenizer'; +import { ITextRangeCollection, IToken, TokenType } from '../language/types'; + +export class LineFormatter { + private builder: TextBuilder; + private tokens: ITextRangeCollection; + private braceCounter: BraceCounter; + private text: string; + + // tslint:disable-next-line:cyclomatic-complexity + public formatLine(text: string): string { + this.tokens = new Tokenizer().tokenize(text); + this.text = text; + this.builder = new TextBuilder(); + this.braceCounter = new BraceCounter(); + + if (this.tokens.count === 0) { + return this.text; + } + + const ws = this.text.substr(0, this.tokens.getItemAt(0).start); + if (ws.length > 0) { + this.builder.append(ws); // Preserve leading indentation + } + + for (let i = 0; i < this.tokens.count; i += 1) { + const t = this.tokens.getItemAt(i); + const prev = i > 0 ? this.tokens.getItemAt(i - 1) : undefined; + const next = i < this.tokens.count - 1 ? this.tokens.getItemAt(i + 1) : undefined; + + switch (t.type) { + case TokenType.Operator: + this.handleOperator(i); + break; + + case TokenType.Comma: + this.builder.append(','); + if (next && !this.isCloseBraceType(next.type)) { + this.builder.softAppendSpace(); + } + break; + + case TokenType.Identifier: + if (!prev || (!this.isOpenBraceType(prev.type) && prev.type !== TokenType.Colon)) { + this.builder.softAppendSpace(); + } + this.builder.append(this.text.substring(t.start, t.end)); + break; + + case TokenType.Colon: + // x: 1 if not in slice, x[1:y] if inside the slice + this.builder.append(':'); + if (!this.braceCounter.isOpened(TokenType.OpenBracket) && (next && next.type !== TokenType.Colon)) { + // Not inside opened [[ ... ] sequence + this.builder.softAppendSpace(); + } + break; + + case TokenType.Comment: + // add space before in-line comment + if (prev) { + this.builder.softAppendSpace(); + } + this.builder.append(this.text.substring(t.start, t.end)); + break; + + default: + this.handleOther(t); + break; + } + } + return this.builder.getText(); + } + + private handleOperator(index: number): void { + const t = this.tokens.getItemAt(index); + if (index >= 2 && t.length === 1 && this.text.charCodeAt(t.start) === Char.Equal) { + if (this.braceCounter.isOpened(TokenType.OpenBrace)) { + // Check if this is = in function arguments. If so, do not + // add spaces around it. + const prev = this.tokens.getItemAt(index - 1); + const prevPrev = this.tokens.getItemAt(index - 2); + if (prev.type === TokenType.Identifier && + (prevPrev.type === TokenType.Comma || prevPrev.type === TokenType.OpenBrace)) { + this.builder.append('='); + return; + } + } + } + this.builder.softAppendSpace(); + this.builder.append(this.text.substring(t.start, t.end)); + this.builder.softAppendSpace(); + } + + private handleOther(t: IToken): void { + if (this.isBraceType(t.type)) { + this.braceCounter.countBrace(t); + } + this.builder.append(this.text.substring(t.start, t.end)); + } + + private isOpenBraceType(type: TokenType): boolean { + return type === TokenType.OpenBrace || type === TokenType.OpenBracket || type === TokenType.OpenCurly; + } + private isCloseBraceType(type: TokenType): boolean { + return type === TokenType.CloseBrace || type === TokenType.CloseBracket || type === TokenType.CloseCurly; + } + private isBraceType(type: TokenType): boolean { + return this.isOpenBraceType(type) || this.isCloseBraceType(type); + } +} diff --git a/src/client/language/braceCounter.ts b/src/client/language/braceCounter.ts new file mode 100644 index 000000000000..30d91b537544 --- /dev/null +++ b/src/client/language/braceCounter.ts @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { IToken, TokenType } from './types'; + +class BracePair { + public readonly openBrace: TokenType; + public readonly closeBrace: TokenType; + + constructor(openBrace: TokenType, closeBrace: TokenType) { + this.openBrace = openBrace; + this.closeBrace = closeBrace; + } +} + +class Stack { + private store: IToken[] = []; + public push(val: IToken) { + this.store.push(val); + } + public pop(): IToken | undefined { + return this.store.pop(); + } + public get length(): number { + return this.store.length; + } +} + +export class BraceCounter { + private readonly bracePairs: BracePair[] = [ + new BracePair(TokenType.OpenBrace, TokenType.CloseBrace), + new BracePair(TokenType.OpenBracket, TokenType.CloseBracket), + new BracePair(TokenType.OpenCurly, TokenType.CloseCurly) + ]; + private braceStacks: Stack[] = [new Stack(), new Stack(), new Stack()]; + + public get count(): number { + let c = 0; + for (const s of this.braceStacks) { + c += s.length; + } + return c; + } + + public isOpened(type: TokenType): boolean { + for (let i = 0; i < this.bracePairs.length; i += 1) { + const pair = this.bracePairs[i]; + if (pair.openBrace === type || pair.closeBrace === type) { + return this.braceStacks[i].length > 0; + } + } + return false; + } + + public countBrace(brace: IToken): boolean { + for (let i = 0; i < this.bracePairs.length; i += 1) { + const pair = this.bracePairs[i]; + if (pair.openBrace === brace.type) { + this.braceStacks[i].push(brace); + return true; + } + if (pair.closeBrace === brace.type) { + if (this.braceStacks[i].length > 0) { + this.braceStacks[i].pop(); + } + return true; + } + } + return false; + } +} diff --git a/src/client/language/characterStream.ts b/src/client/language/characterStream.ts index a95af7ede457..09f3bed33f9d 100644 --- a/src/client/language/characterStream.ts +++ b/src/client/language/characterStream.ts @@ -4,6 +4,7 @@ // tslint:disable-next-line:import-name import Char from 'typescript-char'; +import { isLineBreak, isWhiteSpace } from './characters'; import { TextIterator } from './textIterator'; import { ICharacterStream, ITextIterator } from './types'; @@ -70,11 +71,11 @@ export class CharacterStream implements ICharacterStream { } public isAtWhiteSpace(): boolean { - return this.currentChar <= Char.Space || this.currentChar === 0x200B; // Unicode whitespace + return isWhiteSpace(this.currentChar); } public isAtLineBreak(): boolean { - return this.currentChar === Char.CarriageReturn || this.currentChar === Char.LineFeed; + return isLineBreak(this.currentChar); } public skipLineBreak(): void { diff --git a/src/client/language/characters.ts b/src/client/language/characters.ts new file mode 100644 index 000000000000..e0bc20ba4131 --- /dev/null +++ b/src/client/language/characters.ts @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// tslint:disable-next-line:import-name +import Char from 'typescript-char'; +import { getUnicodeCategory, UnicodeCategory } from './unicode'; + +export function isIdentifierStartChar(ch: number) { + switch (ch) { + // Underscore is explicitly allowed to start an identifier + case Char.Underscore: + return true; + // Characters with the Other_ID_Start property + case 0x1885: + case 0x1886: + case 0x2118: + case 0x212E: + case 0x309B: + case 0x309C: + return true; + default: + break; + } + + const cat = getUnicodeCategory(ch); + switch (cat) { + // Supported categories for starting an identifier + case UnicodeCategory.UppercaseLetter: + case UnicodeCategory.LowercaseLetter: + case UnicodeCategory.TitlecaseLetter: + case UnicodeCategory.ModifierLetter: + case UnicodeCategory.OtherLetter: + case UnicodeCategory.LetterNumber: + return true; + default: + break; + } + return false; +} + +export function isIdentifierChar(ch: number) { + if (isIdentifierStartChar(ch)) { + return true; + } + + switch (ch) { + // Characters with the Other_ID_Continue property + case 0x00B7: + case 0x0387: + case 0x1369: + case 0x136A: + case 0x136B: + case 0x136C: + case 0x136D: + case 0x136E: + case 0x136F: + case 0x1370: + case 0x1371: + case 0x19DA: + return true; + default: + break; + } + + switch (getUnicodeCategory(ch)) { + // Supported categories for continuing an identifier + case UnicodeCategory.NonSpacingMark: + case UnicodeCategory.SpacingCombiningMark: + case UnicodeCategory.DecimalDigitNumber: + case UnicodeCategory.ConnectorPunctuation: + return true; + default: + break; + } + return false; +} + +export function isWhiteSpace(ch: number): boolean { + return ch <= Char.Space || ch === 0x200B; // Unicode whitespace +} + +export function isLineBreak(ch: number): boolean { + return ch === Char.CarriageReturn || ch === Char.LineFeed; +} + +export function isDecimal(ch: number): boolean { + return ch >= Char._0 && ch <= Char._9; +} + +export function isHex(ch: number): boolean { + return isDecimal(ch) || (ch >= Char.a && ch <= Char.f) || (ch >= Char.A && ch <= Char.F); +} + +export function isOctal(ch: number): boolean { + return ch >= Char._0 && ch <= Char._7; +} + +export function isBinary(ch: number): boolean { + return ch === Char._0 || ch === Char._1; +} diff --git a/src/client/language/textBuilder.ts b/src/client/language/textBuilder.ts new file mode 100644 index 000000000000..aebe6187696b --- /dev/null +++ b/src/client/language/textBuilder.ts @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { isWhiteSpace } from './characters'; + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +export class TextBuilder { + private segments: string[] = []; + + public getText(): string { + if (this.isLastWhiteSpace()) { + this.segments.pop(); + } + return this.segments.join(''); + } + + public softAppendSpace(): void { + if (!this.isLastWhiteSpace() && this.segments.length > 0) { + this.segments.push(' '); + } + } + + public append(text: string): void { + this.segments.push(text); + } + + private isLastWhiteSpace(): boolean { + return this.segments.length > 0 && this.isWhitespace(this.segments[this.segments.length - 1]); + } + + private isWhitespace(s: string): boolean { + for (let i = 0; i < s.length; i += 1) { + if (!isWhiteSpace(s.charCodeAt(i))) { + return false; + } + } + return true; + } +} diff --git a/src/client/language/tokenizer.ts b/src/client/language/tokenizer.ts index 60d9fadc7e2e..b84a11b62fd1 100644 --- a/src/client/language/tokenizer.ts +++ b/src/client/language/tokenizer.ts @@ -4,9 +4,10 @@ // tslint:disable-next-line:import-name import Char from 'typescript-char'; +import { isBinary, isDecimal, isHex, isIdentifierChar, isIdentifierStartChar, isOctal } from './characters'; import { CharacterStream } from './characterStream'; import { TextRangeCollection } from './textRangeCollection'; -import { ICharacterStream, ITextRangeCollection, IToken, ITokenizer, TextRange, TokenType } from './types'; +import { ICharacterStream, ITextRangeCollection, IToken, ITokenizer, TextRange, TokenizerMode, TokenType } from './types'; enum QuoteType { None, @@ -26,13 +27,26 @@ class Token extends TextRange implements IToken { } export class Tokenizer implements ITokenizer { + // private keywords = [ + // 'and', 'assert', 'break', 'class', 'continue', 'def', 'del', + // 'elif', 'else', 'except', 'exec', 'False', 'finally', 'for', 'from', + // 'global', 'if', 'import', 'in', 'is', 'lambda', 'None', 'nonlocal', + // 'not', 'or', 'pass', 'print', 'raise', 'return', 'True', 'try', + // 'while', 'with', 'yield' + // ]; private cs: ICharacterStream; private tokens: IToken[] = []; + private floatRegex = /[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?/; + private mode: TokenizerMode; - public Tokenize(text: string): ITextRangeCollection; - public Tokenize(text: string, start: number, length: number): ITextRangeCollection; + constructor() { + //this.floatRegex.compile(); + } + + public tokenize(text: string): ITextRangeCollection; + public tokenize(text: string, start: number, length: number, mode: TokenizerMode): ITextRangeCollection; - public Tokenize(text: string, start?: number, length?: number): ITextRangeCollection { + public tokenize(text: string, start?: number, length?: number, mode?: TokenizerMode): ITextRangeCollection { if (start === undefined) { start = 0; } else if (start < 0 || start >= text.length) { @@ -41,10 +55,12 @@ export class Tokenizer implements ITokenizer { if (length === undefined) { length = text.length; - } else if (length < 0 || start + length >= text.length) { + } else if (length < 0 || start + length > text.length) { throw new Error('Invalid range length'); } + this.mode = mode ? mode : TokenizerMode.Full; + this.cs = new CharacterStream(text); this.cs.position = start; @@ -75,12 +91,229 @@ export class Tokenizer implements ITokenizer { this.handleString(quoteType); return true; } + if (this.cs.currentChar === Char.Hash) { + this.handleComment(); + return true; + } + if (this.mode === TokenizerMode.CommentsAndStrings) { + return false; + } + switch (this.cs.currentChar) { - case Char.Hash: - this.handleComment(); + case Char.OpenParenthesis: + this.tokens.push(new Token(TokenType.OpenBrace, this.cs.position, 1)); + break; + case Char.CloseParenthesis: + this.tokens.push(new Token(TokenType.CloseBrace, this.cs.position, 1)); + break; + case Char.OpenBracket: + this.tokens.push(new Token(TokenType.OpenBracket, this.cs.position, 1)); + break; + case Char.CloseBracket: + this.tokens.push(new Token(TokenType.CloseBracket, this.cs.position, 1)); + break; + case Char.OpenBrace: + this.tokens.push(new Token(TokenType.OpenCurly, this.cs.position, 1)); + break; + case Char.CloseBrace: + this.tokens.push(new Token(TokenType.CloseCurly, this.cs.position, 1)); + break; + case Char.Comma: + this.tokens.push(new Token(TokenType.Comma, this.cs.position, 1)); + break; + case Char.Semicolon: + this.tokens.push(new Token(TokenType.Semicolon, this.cs.position, 1)); + break; + case Char.Colon: + this.tokens.push(new Token(TokenType.Colon, this.cs.position, 1)); break; default: + if (this.isPossibleNumber()) { + if (this.tryNumber()) { + return true; + } + } + if (!this.tryIdentifier()) { + if (!this.tryOperator()) { + this.handleUnknown(); + } + } + return true; + } + return false; + } + + private tryIdentifier(): boolean { + const start = this.cs.position; + if (isIdentifierStartChar(this.cs.currentChar)) { + this.cs.moveNext(); + while (isIdentifierChar(this.cs.currentChar)) { + this.cs.moveNext(); + } + } + if (this.cs.position > start) { + // const text = this.cs.getText().substr(start, this.cs.position - start); + // const type = this.keywords.find((value, index) => value === text) ? TokenType.Keyword : TokenType.Identifier; + this.tokens.push(new Token(TokenType.Identifier, start, this.cs.position - start)); + return true; + } + return false; + } + + private isPossibleNumber(): boolean { + if (this.cs.currentChar === Char.Hyphen || this.cs.currentChar === Char.Plus) { + // Next character must be decimal or a dot otherwise + // it is not a number. No whitespace is allowed. + if (isDecimal(this.cs.nextChar) || this.cs.nextChar === Char.Period) { + // Check what previous token is, if any + if (this.tokens.length === 0) { + // At the start of the file this can only be a number + return true; + } + + const prev = this.tokens[this.tokens.length - 1]; + if (prev.type === TokenType.OpenBrace + || prev.type === TokenType.OpenBracket + || prev.type === TokenType.Comma + || prev.type === TokenType.Semicolon + || prev.type === TokenType.Operator) { + return true; + } + } + return false; + } + + if (isDecimal(this.cs.currentChar)) { + return true; + } + + if (this.cs.currentChar === Char.Period && isDecimal(this.cs.nextChar)) { + return true; + } + + return false; + } + + // tslint:disable-next-line:cyclomatic-complexity + private tryNumber(): boolean { + const start = this.cs.position; + + if (this.cs.currentChar === Char._0) { + let radix = 0; + // Try hex + if (this.cs.nextChar === Char.x || this.cs.nextChar === Char.X) { + this.cs.advance(2); + while (isHex(this.cs.currentChar)) { + this.cs.moveNext(); + } + radix = 16; + } + // Try binary + if (this.cs.nextChar === Char.b || this.cs.nextChar === Char.B) { + this.cs.advance(2); + while (isBinary(this.cs.currentChar)) { + this.cs.moveNext(); + } + radix = 2; + } + // Try octal + if (this.cs.nextChar === Char.o || this.cs.nextChar === Char.O) { + this.cs.advance(2); + while (isOctal(this.cs.currentChar)) { + this.cs.moveNext(); + } + radix = 8; + } + const text = this.cs.getText().substr(start, this.cs.position - start); + if (radix > 0 && parseInt(text.substr(2), radix)) { + this.tokens.push(new Token(TokenType.Number, start, text.length)); + return true; + } + } + + if (isDecimal(this.cs.currentChar) || + this.cs.currentChar === Char.Plus || this.cs.currentChar === Char.Hyphen || this.cs.currentChar === Char.Period) { + const candidate = this.cs.getText().substr(this.cs.position); + const re = this.floatRegex.exec(candidate); + if (re && re.length > 0 && re[0] && candidate.startsWith(re[0])) { + this.tokens.push(new Token(TokenType.Number, start, re[0].length)); + this.cs.position = start + re[0].length; + return true; + } + } + + this.cs.position = start; + return false; + } + + // tslint:disable-next-line:cyclomatic-complexity + private tryOperator(): boolean { + let length = 0; + const nextChar = this.cs.nextChar; + switch (this.cs.currentChar) { + case Char.Plus: + case Char.Hyphen: + case Char.Ampersand: + case Char.Bar: + case Char.Caret: + case Char.Equal: + case Char.ExclamationMark: + length = nextChar === Char.Equal ? 2 : 1; break; + + case Char.Asterisk: + if (nextChar === Char.Asterisk) { + length = this.cs.lookAhead(2) === Char.Equal ? 3 : 2; + } else { + length = nextChar === Char.Equal ? 2 : 1; + } + break; + + case Char.Slash: + if (nextChar === Char.Slash) { + length = this.cs.lookAhead(2) === Char.Equal ? 3 : 2; + } else { + length = nextChar === Char.Equal ? 2 : 1; + } + break; + + case Char.Less: + if (nextChar === Char.Greater) { + length = 2; + } else if (nextChar === Char.Less) { + length = this.cs.lookAhead(2) === Char.Equal ? 3 : 2; + } else { + length = 1; + } + break; + + case Char.Greater: + if (nextChar === Char.Greater) { + length = this.cs.lookAhead(2) === Char.Equal ? 3 : 2; + } else { + length = 1; + } + break; + + case Char.At: + length = nextChar === Char.Equal ? 2 : 0; + break; + + default: + break; + } + this.tokens.push(new Token(TokenType.Operator, this.cs.position, length)); + this.cs.advance(length); + return length > 0; + } + + private handleUnknown(): boolean { + const start = this.cs.position; + this.cs.skipToWhitespace(); + const length = this.cs.position - start; + if (length > 0) { + this.tokens.push(new Token(TokenType.Unknown, start, length)); + return true; } return false; } diff --git a/src/client/language/types.ts b/src/client/language/types.ts index 121ee682c085..76c133a9061c 100644 --- a/src/client/language/types.ts +++ b/src/client/language/types.ts @@ -72,15 +72,34 @@ export interface ICharacterStream extends ITextIterator { } export enum TokenType { + Unknown, String, - Comment + Comment, + Keyword, + Number, + Identifier, + Operator, + Colon, + Semicolon, + Comma, + OpenBrace, + CloseBrace, + OpenBracket, + CloseBracket, + OpenCurly, + CloseCurly } export interface IToken extends ITextRange { readonly type: TokenType; } +export enum TokenizerMode { + CommentsAndStrings, + Full +} + export interface ITokenizer { - Tokenize(text: string): ITextRangeCollection; - Tokenize(text: string, start: number, length: number): ITextRangeCollection; + tokenize(text: string): ITextRangeCollection; + tokenize(text: string, start: number, length: number, mode: TokenizerMode): ITextRangeCollection; } diff --git a/src/client/language/unicode.ts b/src/client/language/unicode.ts new file mode 100644 index 000000000000..6c13a10364a0 --- /dev/null +++ b/src/client/language/unicode.ts @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; + +// tslint:disable-next-line:no-require-imports no-var-requires +const unicodeLu = require('unicode/category/Lu'); +// tslint:disable-next-line:no-require-imports no-var-requires +const unicodeLl = require('unicode/category/Ll'); +// tslint:disable-next-line:no-require-imports no-var-requires +const unicodeLt = require('unicode/category/Lt'); +// tslint:disable-next-line:no-require-imports no-var-requires +const unicodeLo = require('unicode/category/Lo'); +// tslint:disable-next-line:no-require-imports no-var-requires +const unicodeLm = require('unicode/category/Lm'); +// tslint:disable-next-line:no-require-imports no-var-requires +const unicodeNl = require('unicode/category/Nl'); +// tslint:disable-next-line:no-require-imports no-var-requires +const unicodeMn = require('unicode/category/Mn'); +// tslint:disable-next-line:no-require-imports no-var-requires +const unicodeMc = require('unicode/category/Mc'); +// tslint:disable-next-line:no-require-imports no-var-requires +const unicodeNd = require('unicode/category/Nd'); +// tslint:disable-next-line:no-require-imports no-var-requires +const unicodePc = require('unicode/category/Pc'); + +export enum UnicodeCategory { + Unknown, + UppercaseLetter, + LowercaseLetter, + TitlecaseLetter, + ModifierLetter, + OtherLetter, + LetterNumber, + NonSpacingMark, + SpacingCombiningMark, + DecimalDigitNumber, + ConnectorPunctuation +} + +export function getUnicodeCategory(ch: number): UnicodeCategory { + if (unicodeLu[ch]) { + return UnicodeCategory.UppercaseLetter; + } + if (unicodeLl[ch]) { + return UnicodeCategory.LowercaseLetter; + } + if (unicodeLt[ch]) { + return UnicodeCategory.TitlecaseLetter; + } + if (unicodeLo[ch]) { + return UnicodeCategory.OtherLetter; + } + if (unicodeLm[ch]) { + return UnicodeCategory.ModifierLetter; + } + if (unicodeNl[ch]) { + return UnicodeCategory.LetterNumber; + } + if (unicodeMn[ch]) { + return UnicodeCategory.NonSpacingMark; + } + if (unicodeMc[ch]) { + return UnicodeCategory.SpacingCombiningMark; + } + if (unicodeNd[ch]) { + return UnicodeCategory.DecimalDigitNumber; + } + if (unicodePc[ch]) { + return UnicodeCategory.ConnectorPunctuation; + } + return UnicodeCategory.Unknown; +} diff --git a/src/client/providers/completionSource.ts b/src/client/providers/completionSource.ts index d238b9c5636c..3496f38a6f99 100644 --- a/src/client/providers/completionSource.ts +++ b/src/client/providers/completionSource.ts @@ -4,11 +4,10 @@ import * as vscode from 'vscode'; import { PythonSettings } from '../common/configSettings'; -import { Tokenizer } from '../language/tokenizer'; -import { TokenType } from '../language/types'; import { JediFactory } from '../languageServices/jediProxyFactory'; import { ItemInfoSource, LanguageItemInfo } from './itemInfoSource'; import * as proxy from './jediProxy'; +import { isPositionInsideStringOrComment } from './providerUtilities'; class DocumentPosition { constructor(public document: vscode.TextDocument, public position: vscode.Position) { } @@ -76,7 +75,7 @@ export class CompletionSource { return undefined; } // Suppress completion inside string and comments. - if (this.isPositionInsideStringOrComment(document, position)) { + if (isPositionInsideStringOrComment(document, position)) { return undefined; } const type = proxy.CommandType.Completions; @@ -110,17 +109,4 @@ export class CompletionSource { documentPosition.attachTo(completionItem); return completionItem; } - - private isPositionInsideStringOrComment(document: vscode.TextDocument, position: vscode.Position): boolean { - const tokenizeTo = position.translate(1, 0); - const text = document.getText(new vscode.Range(new vscode.Position(0, 0), tokenizeTo)); - const t = new Tokenizer(); - const tokens = t.Tokenize(text); - const index = tokens.getItemContaining(document.offsetAt(position)); - if (index >= 0) { - const token = tokens.getItemAt(index); - return token.type === TokenType.String || token.type === TokenType.Comment; - } - return false; - } } diff --git a/src/client/providers/providerUtilities.ts b/src/client/providers/providerUtilities.ts new file mode 100644 index 000000000000..4b113c61eceb --- /dev/null +++ b/src/client/providers/providerUtilities.ts @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as vscode from 'vscode'; +import { Tokenizer } from '../language/tokenizer'; +import { ITextRangeCollection, IToken, TokenizerMode, TokenType } from '../language/types'; + +export function getDocumentTokens(document: vscode.TextDocument, tokenizeTo: vscode.Position, mode: TokenizerMode): ITextRangeCollection { + const text = document.getText(new vscode.Range(new vscode.Position(0, 0), tokenizeTo)); + return new Tokenizer().tokenize(text, 0, text.length, mode); +} + +export function isPositionInsideStringOrComment(document: vscode.TextDocument, position: vscode.Position): boolean { + const tokenizeTo = position.translate(1, 0); + const tokens = getDocumentTokens(document, tokenizeTo, TokenizerMode.CommentsAndStrings); + const index = tokens.getItemContaining(document.offsetAt(position)); + if (index >= 0) { + const token = tokens.getItemAt(index); + return token.type === TokenType.String || token.type === TokenType.Comment; + } + return false; +} diff --git a/src/client/typeFormatters/blockFormatProvider.ts b/src/client/typeFormatters/blockFormatProvider.ts index f0fc911c6afd..415d8692e7d1 100644 --- a/src/client/typeFormatters/blockFormatProvider.ts +++ b/src/client/typeFormatters/blockFormatProvider.ts @@ -1,9 +1,9 @@ -import { OnTypeFormattingEditProvider, FormattingOptions, TextEdit, CancellationToken, TextDocument } from 'vscode'; +import { CancellationToken, FormattingOptions, OnTypeFormattingEditProvider, TextDocument, TextEdit } from 'vscode'; import { Position } from 'vscode'; import { CodeBlockFormatProvider } from './codeBlockFormatProvider'; -import { IF_REGEX, ELIF_REGEX, ELSE_REGEX, FOR_IN_REGEX, ASYNC_FOR_IN_REGEX, WHILE_REGEX } from './contracts'; -import { TRY_REGEX, FINALLY_REGEX, EXCEPT_REGEX } from './contracts'; -import { DEF_REGEX, ASYNC_DEF_REGEX, CLASS_REGEX } from './contracts'; +import { ASYNC_FOR_IN_REGEX, ELIF_REGEX, ELSE_REGEX, FOR_IN_REGEX, IF_REGEX, WHILE_REGEX } from './contracts'; +import { EXCEPT_REGEX, FINALLY_REGEX, TRY_REGEX } from './contracts'; +import { ASYNC_DEF_REGEX, CLASS_REGEX, DEF_REGEX } from './contracts'; export class BlockFormatProviders implements OnTypeFormattingEditProvider { private providers: CodeBlockFormatProvider[]; @@ -44,7 +44,8 @@ export class BlockFormatProviders implements OnTypeFormattingEditProvider { ]; this.providers.push(new CodeBlockFormatProvider(FINALLY_REGEX, finallyParentBlocks, boundaryBlocks)); } - provideOnTypeFormattingEdits(document: TextDocument, position: Position, ch: string, options: FormattingOptions, token: CancellationToken): TextEdit[] { + + public provideOnTypeFormattingEdits(document: TextDocument, position: Position, ch: string, options: FormattingOptions, token: CancellationToken): TextEdit[] { if (position.line === 0) { return []; } @@ -59,11 +60,11 @@ export class BlockFormatProviders implements OnTypeFormattingEditProvider { } const currentLineText = currentLine.text; - const provider = this.providers.find(provider => provider.canProvideEdits(currentLineText)); + const provider = this.providers.find(p => p.canProvideEdits(currentLineText)); if (provider) { return provider.provideEdits(document, position, ch, options, currentLine); } return []; } -} \ No newline at end of file +} diff --git a/src/client/typeFormatters/codeBlockFormatProvider.ts b/src/client/typeFormatters/codeBlockFormatProvider.ts index 179de4b8ab94..d7518ad8b419 100644 --- a/src/client/typeFormatters/codeBlockFormatProvider.ts +++ b/src/client/typeFormatters/codeBlockFormatProvider.ts @@ -1,15 +1,15 @@ -import { FormattingOptions, TextEdit, TextDocument } from 'vscode'; +import { FormattingOptions, TextDocument, TextEdit } from 'vscode'; import { Position, Range, TextLine } from 'vscode'; import { BlockRegEx } from './contracts'; export class CodeBlockFormatProvider { constructor(private blockRegExp: BlockRegEx, private previousBlockRegExps: BlockRegEx[], private boundaryRegExps: BlockRegEx[]) { } - canProvideEdits(line: string): boolean { + public canProvideEdits(line: string): boolean { return this.blockRegExp.test(line); } - provideEdits(document: TextDocument, position: Position, ch: string, options: FormattingOptions, line: TextLine): TextEdit[] { + public provideEdits(document: TextDocument, position: Position, ch: string, options: FormattingOptions, line: TextLine): TextEdit[] { // We can have else for the following blocks: // if: // elif x: @@ -17,7 +17,7 @@ export class CodeBlockFormatProvider { // while x: // We need to find a block statement that is less than or equal to this statement block (but not greater) - for (let lineNumber = position.line - 1; lineNumber >= 0; lineNumber--) { + for (let lineNumber = position.line - 1; lineNumber >= 0; lineNumber -= 1) { const prevLine = document.lineAt(lineNumber); const prevLineText = prevLine.text; @@ -44,12 +44,12 @@ export class CodeBlockFormatProvider { // current block cannot be at the same level as a preivous block continue; } + if (options.insertSpaces) { return [ TextEdit.delete(new Range(startPosition, endPosition)) ]; - } - else { + } else { // Delete everything before the block and insert the same characters we have in the previous block const prefixOfPreviousBlock = prevLineText.substring(0, startOfBlockInLine); @@ -65,4 +65,4 @@ export class CodeBlockFormatProvider { return []; } -} \ No newline at end of file +} diff --git a/src/client/typeFormatters/onEnterFormatter.ts b/src/client/typeFormatters/onEnterFormatter.ts new file mode 100644 index 000000000000..013b2d2a85f9 --- /dev/null +++ b/src/client/typeFormatters/onEnterFormatter.ts @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as vscode from 'vscode'; +import { LineFormatter } from '../formatters/lineFormatter'; +import { TokenizerMode, TokenType } from '../language/types'; +import { getDocumentTokens } from '../providers/providerUtilities'; + +export class OnEnterFormatter implements vscode.OnTypeFormattingEditProvider { + private readonly formatter = new LineFormatter(); + + public provideOnTypeFormattingEdits( + document: vscode.TextDocument, + position: vscode.Position, + ch: string, + options: vscode.FormattingOptions, + cancellationToken: vscode.CancellationToken): vscode.TextEdit[] { + if (position.line === 0) { + return []; + } + + // Check case when the entire line belongs to a comment or string + const prevLine = document.lineAt(position.line - 1); + const tokens = getDocumentTokens(document, position, TokenizerMode.CommentsAndStrings); + const lineStartTokenIndex = tokens.getItemContaining(document.offsetAt(prevLine.range.start)); + const lineEndTokenIndex = tokens.getItemContaining(document.offsetAt(prevLine.range.end)); + if (lineStartTokenIndex >= 0 && lineStartTokenIndex === lineEndTokenIndex) { + const token = tokens.getItemAt(lineStartTokenIndex); + if (token.type === TokenType.Semicolon || token.type === TokenType.String) { + return []; + } + } + const formatted = this.formatter.formatLine(prevLine.text); + if (formatted === prevLine.text) { + return []; + } + return [new vscode.TextEdit(prevLine.range, formatted)]; + } +} diff --git a/src/test/format/extension.lineFormatter.test.ts b/src/test/format/extension.lineFormatter.test.ts new file mode 100644 index 000000000000..842cb02d735d --- /dev/null +++ b/src/test/format/extension.lineFormatter.test.ts @@ -0,0 +1,68 @@ + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as assert from 'assert'; +import { LineFormatter } from '../../client/formatters/lineFormatter'; + +// https://www.python.org/dev/peps/pep-0008/#code-lay-out +suite('Formatting - line formatter', () => { + const formatter = new LineFormatter(); + + test('Operator spacing', () => { + const actual = formatter.formatLine('( x +1 )*y/ 3'); + assert.equal(actual, '(x + 1) * y / 3'); + }); + test('Braces spacing', () => { + const actual = formatter.formatLine('foo =(0 ,)'); + assert.equal(actual, 'foo = (0,)'); + }); + test('Function arguments', () => { + const actual = formatter.formatLine('foo (0 , x= 1, (3+7) , y , z )'); + assert.equal(actual, 'foo(0, x=1, (3 + 7), y, z)'); + }); + test('Colon regular', () => { + const actual = formatter.formatLine('if x == 4 : print x,y; x,y= y, x'); + assert.equal(actual, 'if x == 4: print x, y; x, y = y, x'); + }); + test('Colon slices', () => { + const actual = formatter.formatLine('x[1: 30]'); + assert.equal(actual, 'x[1:30]'); + }); + test('Colon slices in arguments', () => { + const actual = formatter.formatLine('spam ( ham[ 1 :3], {eggs : 2})'); + assert.equal(actual, 'spam(ham[1:3], {eggs: 2})'); + }); + test('Colon slices with double colon', () => { + const actual = formatter.formatLine('ham [1:9 ], ham[ 1: 9: 3], ham[: 9 :3], ham[1: :3], ham [ 1: 9:]'); + assert.equal(actual, 'ham[1:9], ham[1:9:3], ham[:9:3], ham[1::3], ham[1:9:]'); + }); + test('Colon slices with operators', () => { + const actual = formatter.formatLine('ham [lower+ offset :upper+offset]'); + assert.equal(actual, 'ham[lower + offset:upper + offset]'); + }); + test('Colon slices with functions', () => { + const actual = formatter.formatLine('ham[ : upper_fn ( x) : step_fn(x )], ham[ :: step_fn(x)]'); + assert.equal(actual, 'ham[:upper_fn(x):step_fn(x)], ham[::step_fn(x)]'); + }); + test('Colon in for loop', () => { + const actual = formatter.formatLine('for index in range( len(fruits) ): '); + assert.equal(actual, 'for index in range(len(fruits)):'); + }); + test('Nested braces', () => { + const actual = formatter.formatLine('[ 1 :[2: (x,),y]]{1}'); + assert.equal(actual, '[1:[2:(x,), y]]{1}'); + }); + test('Trailing comment', () => { + const actual = formatter.formatLine('x=1 # comment'); + assert.equal(actual, 'x = 1 # comment'); + }); + test('Single comment', () => { + const actual = formatter.formatLine('# comment'); + assert.equal(actual, '# comment'); + }); + test('Comment with leading whitespace', () => { + const actual = formatter.formatLine(' # comment'); + assert.equal(actual, ' # comment'); + }); +}); diff --git a/src/test/format/extension.onEnterFormat.test.ts b/src/test/format/extension.onEnterFormat.test.ts new file mode 100644 index 000000000000..1622975a39d2 --- /dev/null +++ b/src/test/format/extension.onEnterFormat.test.ts @@ -0,0 +1,58 @@ + +// Note: This example test is leveraging the Mocha test framework. +// Please refer to their documentation on https://mochajs.org/ for help. + +import * as assert from 'assert'; +import * as path from 'path'; +import * as vscode from 'vscode'; +import { closeActiveWindows, initialize } from '../initialize'; + +const formatFilesPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'formatting'); +const unformattedFile = path.join(formatFilesPath, 'fileToFormatOnEnter.py'); + +suite('Formatting - OnEnter provider', () => { + let document: vscode.TextDocument; + + suiteSetup(initialize); + setup(async () => { + document = await vscode.workspace.openTextDocument(unformattedFile); + await vscode.window.showTextDocument(document); + }); + suiteTeardown(closeActiveWindows); + teardown(closeActiveWindows); + + test('Regular string', async () => { + const edits = await formatAtPosition(1, 0); + assert.notEqual(edits!.length, 0, 'Line was not formatted'); + }); + + test('No formatting inside strings', async () => { + const edits = await formatAtPosition(2, 0); + assert.equal(edits!.length, 0, 'Text inside string was formatted'); + }); + + test('Whitespace before comment', async () => { + const edits = await formatAtPosition(4, 0); + assert.equal(edits!.length, 0, 'Whitespace before comment was formatted'); + }); + + test('No formatting of comment', async () => { + const edits = await formatAtPosition(5, 0); + assert.equal(edits!.length, 0, 'Text inside comment was formatted'); + }); + + test('Formatting line ending in comment', async () => { + const edits = await formatAtPosition(6, 0); + assert.notEqual(edits!.length, 0, 'Line ending in comment was not formatted'); + }); + + test('Formatting line ending in string', async () => { + const edits = await formatAtPosition(7, 0); + assert.notEqual(edits!.length, 0, 'Line ending in multilint string was not formatted'); + }); + + async function formatAtPosition(line: number, character: number): Promise { + return await vscode.commands.executeCommand('vscode.executeFormatOnTypeProvider', + document.uri, new vscode.Position(line, character), '\n', { insertSpaces: true, tabSize: 2 }); + } +}); diff --git a/src/test/format/extension.onTypeFormat.test.ts b/src/test/format/extension.onTypeFormat.test.ts index 097922e9010e..06f449323836 100644 --- a/src/test/format/extension.onTypeFormat.test.ts +++ b/src/test/format/extension.onTypeFormat.test.ts @@ -2,17 +2,12 @@ // Note: This example test is leveraging the Mocha test framework. // Please refer to their documentation on https://mochajs.org/ for help. - -// The module 'assert' provides assertion methods from node import * as assert from 'assert'; - -// You can import and use all API from the 'vscode' module -// as well as import your extension to test it -import * as vscode from 'vscode'; -import * as path from 'path'; import * as fs from 'fs-extra'; -import { initialize, closeActiveWindows, initializeTest } from '../initialize'; +import * as path from 'path'; +import * as vscode from 'vscode'; import { BlockFormatProviders } from '../../client/typeFormatters/blockFormatProvider'; +import { closeActiveWindows, initialize, initializeTest } from '../initialize'; const srcPythoFilesPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'typeFormatFiles'); const outPythoFilesPath = path.join(__dirname, 'pythonFiles', 'typeFormatFiles'); @@ -37,7 +32,8 @@ function testFormatting(fileToFormat: string, position: vscode.Position, expecte textDocument = document; return vscode.window.showTextDocument(textDocument); }).then(editor => { - return provider.provideOnTypeFormattingEdits(textDocument, position, ':', formatOptions, null); + return provider.provideOnTypeFormattingEdits( + textDocument, position, ':', formatOptions, new vscode.CancellationTokenSource().token); }).then(edits => { assert.equal(edits.length, expectedEdits.length, 'Number of edits not the same'); edits.forEach((edit, index) => { @@ -52,7 +48,6 @@ function testFormatting(fileToFormat: string, position: vscode.Position, expecte }); } - suite('Else block with if in first line of file', () => { suiteSetup(async () => { await initialize(); @@ -64,10 +59,10 @@ suite('Else block with if in first line of file', () => { fs.copySync(path.join(srcPythoFilesPath, file), targetFile); }); }); - suiteTeardown(() => closeActiveWindows()); - teardown(() => closeActiveWindows()); + suiteTeardown(closeActiveWindows); + teardown(closeActiveWindows); - interface TestCase { + interface ITestCase { title: string; line: number; column: number; @@ -75,8 +70,7 @@ suite('Else block with if in first line of file', () => { formatOptions: vscode.FormattingOptions; filePath: string; } - const TAB = ' '; - const testCases: TestCase[] = [ + const testCases: ITestCase[] = [ { title: 'else block with 2 spaces', line: 3, column: 7, @@ -126,16 +120,16 @@ suite('Try blocks with indentation of 2 spaces', () => { fs.copySync(path.join(srcPythoFilesPath, file), targetFile); }); }); - suiteTeardown(() => closeActiveWindows()); - teardown(() => closeActiveWindows()); + suiteTeardown(closeActiveWindows); + teardown(closeActiveWindows); - interface TestCase { + interface ITestCase { title: string; line: number; column: number; expectedEdits: vscode.TextEdit[]; } - const testCases: TestCase[] = [ + const testCases: ITestCase[] = [ { title: 'except off by tab', line: 6, column: 22, @@ -227,16 +221,16 @@ suite('Try blocks with indentation of 4 spaces', () => { fs.copySync(path.join(srcPythoFilesPath, file), targetFile); }); }); - suiteTeardown(() => closeActiveWindows()); - teardown(() => closeActiveWindows()); + suiteTeardown(closeActiveWindows); + teardown(closeActiveWindows); - interface TestCase { + interface ITestCase { title: string; line: number; column: number; expectedEdits: vscode.TextEdit[]; } - const testCases: TestCase[] = [ + const testCases: ITestCase[] = [ { title: 'except off by tab', line: 6, column: 22, @@ -328,17 +322,17 @@ suite('Try blocks with indentation of Tab', () => { fs.copySync(path.join(srcPythoFilesPath, file), targetFile); }); }); - suiteTeardown(() => closeActiveWindows()); - teardown(() => closeActiveWindows()); + suiteTeardown(closeActiveWindows); + teardown(closeActiveWindows); - interface TestCase { + interface ITestCase { title: string; line: number; column: number; expectedEdits: vscode.TextEdit[]; } const TAB = ' '; - const testCases: TestCase[] = [ + const testCases: ITestCase[] = [ { title: 'except off by tab', line: 6, column: 22, @@ -409,6 +403,7 @@ suite('Try blocks with indentation of Tab', () => { }); }); +// tslint:disable-next-line:max-func-body-length suite('Else blocks with indentation of 2 spaces', () => { suiteSetup(async () => { await initialize(); @@ -420,9 +415,10 @@ suite('Else blocks with indentation of 2 spaces', () => { fs.copySync(path.join(srcPythoFilesPath, file), targetFile); }); }); - suiteTeardown(() => closeActiveWindows()); - teardown(() => closeActiveWindows()); + suiteTeardown(closeActiveWindows); + teardown(closeActiveWindows); + // tslint:disable-next-line:interface-name interface TestCase { title: string; line: number; @@ -540,6 +536,7 @@ suite('Else blocks with indentation of 2 spaces', () => { }); }); +// tslint:disable-next-line:max-func-body-length suite('Else blocks with indentation of 4 spaces', () => { suiteSetup(async () => { await initialize(); @@ -551,16 +548,16 @@ suite('Else blocks with indentation of 4 spaces', () => { fs.copySync(path.join(srcPythoFilesPath, file), targetFile); }); }); - suiteTeardown(() => closeActiveWindows()); - teardown(() => closeActiveWindows()); + suiteTeardown(closeActiveWindows); + teardown(closeActiveWindows); - interface TestCase { + interface ITestCase { title: string; line: number; column: number; expectedEdits: vscode.TextEdit[]; } - const testCases: TestCase[] = [ + const testCases: ITestCase[] = [ { title: 'elif off by tab', line: 4, column: 18, @@ -665,6 +662,7 @@ suite('Else blocks with indentation of 4 spaces', () => { }); }); +// tslint:disable-next-line:max-func-body-length suite('Else blocks with indentation of Tab', () => { suiteSetup(async () => { await initialize(); @@ -676,17 +674,17 @@ suite('Else blocks with indentation of Tab', () => { fs.copySync(path.join(srcPythoFilesPath, file), targetFile); }); }); - setup(() => initializeTest()); - suiteTeardown(() => closeActiveWindows()); - teardown(() => closeActiveWindows()); + setup(initializeTest); + suiteTeardown(closeActiveWindows); + teardown(closeActiveWindows); - interface TestCase { + interface ITestCase { title: string; line: number; column: number; expectedEdits: vscode.TextEdit[]; } - const testCases: TestCase[] = [ + const testCases: ITestCase[] = [ { title: 'elif off by tab', line: 4, column: 18, diff --git a/src/test/language/tokenizer.test.ts b/src/test/language/tokenizer.test.ts index cef3b08e5fb1..727ce969dd09 100644 --- a/src/test/language/tokenizer.test.ts +++ b/src/test/language/tokenizer.test.ts @@ -5,20 +5,20 @@ import * as assert from 'assert'; import { TextRangeCollection } from '../../client/language/textRangeCollection'; import { Tokenizer } from '../../client/language/tokenizer'; -import { TextRange, TokenType } from '../../client/language/types'; +import { TokenType } from '../../client/language/types'; // tslint:disable-next-line:max-func-body-length suite('Language.Tokenizer', () => { test('Empty', async () => { const t = new Tokenizer(); - const tokens = t.Tokenize(''); + const tokens = t.tokenize(''); assert.equal(tokens instanceof TextRangeCollection, true); assert.equal(tokens.count, 0); assert.equal(tokens.length, 0); }); test('Strings: unclosed', async () => { const t = new Tokenizer(); - const tokens = t.Tokenize(' "string" """line1\n#line2"""\t\'un#closed'); + const tokens = t.tokenize(' "string" """line1\n#line2"""\t\'un#closed'); assert.equal(tokens.count, 3); const ranges = [1, 8, 10, 18, 29, 10]; @@ -30,7 +30,7 @@ suite('Language.Tokenizer', () => { }); test('Strings: block next to regular, double-quoted', async () => { const t = new Tokenizer(); - const tokens = t.Tokenize('"string""""s2"""'); + const tokens = t.tokenize('"string""""s2"""'); assert.equal(tokens.count, 2); const ranges = [0, 8, 8, 8]; @@ -42,7 +42,7 @@ suite('Language.Tokenizer', () => { }); test('Strings: block next to block, double-quoted', async () => { const t = new Tokenizer(); - const tokens = t.Tokenize('""""""""'); + const tokens = t.tokenize('""""""""'); assert.equal(tokens.count, 2); const ranges = [0, 6, 6, 2]; @@ -54,7 +54,7 @@ suite('Language.Tokenizer', () => { }); test('Strings: unclosed sequence of quotes', async () => { const t = new Tokenizer(); - const tokens = t.Tokenize('"""""'); + const tokens = t.tokenize('"""""'); assert.equal(tokens.count, 1); const ranges = [0, 5]; @@ -66,7 +66,7 @@ suite('Language.Tokenizer', () => { }); test('Comments', async () => { const t = new Tokenizer(); - const tokens = t.Tokenize(' #co"""mment1\n\t\n#comm\'ent2 '); + const tokens = t.tokenize(' #co"""mment1\n\t\n#comm\'ent2 '); assert.equal(tokens.count, 2); const ranges = [1, 12, 15, 11]; diff --git a/src/test/pythonFiles/formatting/fileToFormatOnEnter.py b/src/test/pythonFiles/formatting/fileToFormatOnEnter.py new file mode 100644 index 000000000000..6f680ad49b0d --- /dev/null +++ b/src/test/pythonFiles/formatting/fileToFormatOnEnter.py @@ -0,0 +1,7 @@ +x=1 +"""x=1 +""" + # comment +# x=1 +x+1 # +x+""" diff --git a/tslint.json b/tslint.json index b288b7017c41..4701c1abcbbc 100644 --- a/tslint.json +++ b/tslint.json @@ -25,6 +25,7 @@ "check-else" ], "no-parameter-properties": false, + "no-parameter-reassignment": false, "no-reserved-keywords": false, "newline-before-return": false, "export-name": false, @@ -51,6 +52,7 @@ "no-floating-promises": true, "no-empty-interface": false, "no-bitwise": false, - "eofline": true + "eofline": true, + "switch-final-break": false } }