From b2372c9e7b12fcccc9675485b569e8ec3b1d730c Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Tue, 23 Sep 2014 12:23:33 -0700 Subject: [PATCH 1/7] Fixed up colorization tests. --- .../cases/unittests/services/colorization.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/cases/unittests/services/colorization.ts b/tests/cases/unittests/services/colorization.ts index 7b8ee376064..f86ee747eb4 100644 --- a/tests/cases/unittests/services/colorization.ts +++ b/tests/cases/unittests/services/colorization.ts @@ -36,7 +36,7 @@ describe('Colorization', function () { } var finalEndOfLineState = classResult[classResult.length - 1]; - assert.equal(position, code.length, "Expected accumilative length of all entries to match the length of the source. expected: " + code.length + ", but got: " + position); + assert.equal(position, code.length, "Expected cumulative length of all entries to match the length of the source. expected: " + code.length + ", but got: " + position); return { tuples: tuples, @@ -84,8 +84,8 @@ describe('Colorization', function () { var actualEntry = getEntryAtPosistion(result, actualEntryPosition); assert(actualEntry, "Could not find classification entry for '" + expectedEntry.value + "' at position: " + actualEntryPosition); - assert.equal(actualEntry.length, expectedEntry.value.length, "Classification class does not match expected."); - assert.equal(actualEntry.class, expectedEntry.class, "Classification class does not match expected."); + assert.equal(actualEntry.class, expectedEntry.class, "Classification class does not match expected. Expected: " + ts.TokenClass[expectedEntry.class] + ", Actual: " + ts.TokenClass[actualEntry.class]); + assert.equal(actualEntry.length, expectedEntry.value.length, "Classification length does not match expected. Expected: " + ts.TokenClass[expectedEntry.value.length] + ", Actual: " + ts.TokenClass[actualEntry.length]); } } } @@ -105,7 +105,7 @@ describe('Colorization', function () { punctuation(";")); }); - it("classifies correctelly a comment after a divide operator", function () { + it("classifies correctly a comment after a divide operator", function () { test("1 / 2 // comment", ts.EndOfLineState.Start, numberLiteral("1"), @@ -115,7 +115,7 @@ describe('Colorization', function () { comment("// comment")); }); - it("classifies correctelly a literal after a divide operator", function () { + it("classifies correctly a literal after a divide operator", function () { test("1 / 2, 3 / 4", ts.EndOfLineState.Start, numberLiteral("1"), @@ -127,42 +127,42 @@ describe('Colorization', function () { operator(",")); }); - it("classifies correctelly an unterminated multi-line string", function () { + it("classifies correctly an unterminated multi-line string", function () { test("'line1\\", ts.EndOfLineState.Start, stringLiteral("'line1\\"), finalEndOfLineState(ts.EndOfLineState.InSingleQuoteStringLiteral)); }); - it("classifies correctelly the second line of an unterminated multi-line string", function () { + it("classifies correctly the second line of an unterminated multi-line string", function () { test("\\", ts.EndOfLineState.InDoubleQuoteStringLiteral, stringLiteral("\\"), finalEndOfLineState(ts.EndOfLineState.InDoubleQuoteStringLiteral)); }); - it("classifies correctelly the last line of a multi-line string", function () { + it("classifies correctly the last line of a multi-line string", function () { test("'", ts.EndOfLineState.InSingleQuoteStringLiteral, stringLiteral("'"), finalEndOfLineState(ts.EndOfLineState.Start)); }); - it("classifies correctelly an unterminated multiline comment", function () { + it("classifies correctly an unterminated multiline comment", function () { test("/*", ts.EndOfLineState.Start, comment("/*"), finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); }); - it("classifies correctelly an unterminated multiline comment with trailing space", function () { + it("classifies correctly an unterminated multiline comment with trailing space", function () { test("/* ", ts.EndOfLineState.Start, comment("/* "), finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); }); - it("classifies correctelly a keyword after a dot", function () { + it("classifies correctly a keyword after a dot", function () { test("a.var", ts.EndOfLineState.Start, identifier("var")); From 292cfc87211b9e4c0f6a6732c84fae1552eff483 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Tue, 23 Sep 2014 12:25:57 -0700 Subject: [PATCH 2/7] Use the 'skipTrivia' scanner flag for lexical classification. --- src/compiler/types.ts | 4 +++- src/services/services.ts | 38 +++++++++++++------------------------- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/src/compiler/types.ts b/src/compiler/types.ts index d9c2a420fb6..9165c55e408 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -228,7 +228,9 @@ module ts { FirstPunctuation = OpenBraceToken, LastPunctuation = CaretEqualsToken, FirstToken = EndOfFileToken, - LastToken = StringKeyword + LastToken = StringKeyword, + FirstTriviaToken = SingleLineCommentTrivia, + LastTriviaToken = WhitespaceTrivia } export enum NodeFlags { diff --git a/src/services/services.ts b/src/services/services.ts index 04e63f6dbe1..8fb7e82a82a 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -4076,7 +4076,7 @@ module ts { function getClassificationsForLine(text: string, lexState: EndOfLineState): ClassificationResult { var offset = 0; var lastTokenOrCommentEnd = 0; - var lastToken = SyntaxKind.Unknown; + var lastNonTriviaToken = SyntaxKind.Unknown; var inUnterminatedMultiLineComment = false; // If we're in a string literal, then prepend: "\ @@ -4104,22 +4104,25 @@ module ts { entries: [] }; - scanner = createScanner(ScriptTarget.ES5, /*skipTrivia*/ true, text, onError, processComment); + scanner = createScanner(ScriptTarget.ES5, /*skipTrivia*/ false, text, onError, /*onComment*/ undefined); var token = SyntaxKind.Unknown; do { token = scanner.scan(); - if ((token === SyntaxKind.SlashToken || token === SyntaxKind.SlashEqualsToken) && !noRegexTable[lastToken]) { + if ((token === SyntaxKind.SlashToken || token === SyntaxKind.SlashEqualsToken) && !noRegexTable[lastNonTriviaToken]) { if (scanner.reScanSlashToken() === SyntaxKind.RegularExpressionLiteral) { token = SyntaxKind.RegularExpressionLiteral; } } - else if (lastToken === SyntaxKind.DotToken) { + else if (lastNonTriviaToken === SyntaxKind.DotToken) { token = SyntaxKind.Identifier; } - lastToken = token; + // Only recall the token if it was *not* trivia. + if (!(SyntaxKind.FirstTriviaToken <= token && token <= SyntaxKind.LastTriviaToken)) { + lastNonTriviaToken = token; + } processToken(); } @@ -4132,21 +4135,10 @@ module ts { inUnterminatedMultiLineComment = message.key === Diagnostics.Asterisk_Slash_expected.key; } - function processComment(start: number, end: number) { - // add Leading white spaces - addLeadingWhiteSpace(start, end); - - // add the comment - addResult(end - start, TokenClass.Comment); - } - function processToken(): void { var start = scanner.getTokenPos(); var end = scanner.getTextPos(); - // add Leading white spaces - addLeadingWhiteSpace(start, end); - // add the token addResult(end - start, classFromKind(token)); @@ -4167,15 +4159,6 @@ module ts { } } - function addLeadingWhiteSpace(start: number, end: number): void { - if (start > lastTokenOrCommentEnd) { - addResult(start - lastTokenOrCommentEnd, TokenClass.Whitespace); - } - - // Remember the end of the last token - lastTokenOrCommentEnd = end; - } - function addResult(length: number, classification: TokenClass): void { if (length > 0) { // If this is the first classification we're adding to the list, then remove any @@ -4268,6 +4251,11 @@ module ts { return TokenClass.StringLiteral; case SyntaxKind.RegularExpressionLiteral: return TokenClass.RegExpLiteral; + case SyntaxKind.MultiLineCommentTrivia: + case SyntaxKind.SingleLineCommentTrivia: + return TokenClass.Comment; + case SyntaxKind.WhitespaceTrivia: + return TokenClass.Whitespace; case SyntaxKind.Identifier: default: return TokenClass.Identifier; From d93c28efb2231a9dfcad412530363191a446bb2d Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Tue, 23 Sep 2014 13:31:09 -0700 Subject: [PATCH 3/7] Process unterminated multiline comments the same way we handle multiline strings. --- src/services/services.ts | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 8fb7e82a82a..3afb72ac10a 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -4077,8 +4077,7 @@ module ts { var offset = 0; var lastTokenOrCommentEnd = 0; var lastNonTriviaToken = SyntaxKind.Unknown; - var inUnterminatedMultiLineComment = false; - + // If we're in a string literal, then prepend: "\ // (and a newline). That way when we lex we'll think we're still in a string literal. // @@ -4104,7 +4103,7 @@ module ts { entries: [] }; - scanner = createScanner(ScriptTarget.ES5, /*skipTrivia*/ false, text, onError, /*onComment*/ undefined); + scanner = createScanner(ScriptTarget.ES5, /*skipTrivia*/ false, text, /*onError*/ undefined, /*onComment*/ undefined); var token = SyntaxKind.Unknown; do { @@ -4130,11 +4129,6 @@ module ts { return result; - - function onError(message: DiagnosticMessage): void { - inUnterminatedMultiLineComment = message.key === Diagnostics.Asterisk_Slash_expected.key; - } - function processToken(): void { var start = scanner.getTokenPos(); var end = scanner.getTextPos(); @@ -4144,10 +4138,8 @@ module ts { if (end >= text.length) { // We're at the end. - if (inUnterminatedMultiLineComment) { - result.finalLexState = EndOfLineState.InMultiLineCommentTrivia; - } - else if (token === SyntaxKind.StringLiteral) { + if (token === SyntaxKind.StringLiteral) { + // Check to see if we finished up on a multiline string literal. var tokenText = scanner.getTokenText(); if (tokenText.length > 0 && tokenText.charCodeAt(tokenText.length - 1) === CharacterCodes.backslash) { var quoteChar = tokenText.charCodeAt(0); @@ -4156,6 +4148,15 @@ module ts { : EndOfLineState.InSingleQuoteStringLiteral; } } + else if (token === SyntaxKind.MultiLineCommentTrivia) { + // Check to see if the multiline comment was unclosed. + var tokenText = scanner.getTokenText() + if (!(tokenText.length > 3 && // need to avoid catching '/*/' + tokenText.charCodeAt(tokenText.length - 2) === CharacterCodes.asterisk && + tokenText.charCodeAt(tokenText.length - 1) === CharacterCodes.slash)) { + result.finalLexState = EndOfLineState.InMultiLineCommentTrivia; + } + } } } From 1c73189fa1430b14c90f7a48a3f042d42e3143b8 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Tue, 23 Sep 2014 13:54:56 -0700 Subject: [PATCH 4/7] No longer create an entire scanner when performing line classification. --- src/services/services.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 3afb72ac10a..999c31772e7 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -4050,13 +4050,13 @@ module ts { /// Classifier export function createClassifier(host: Logger): Classifier { - var scanner: Scanner; - var noRegexTable: boolean[]; + var scanner = createScanner(ScriptTarget.ES5, /*skipTrivia*/ false); /// We do not have a full parser support to know when we should parse a regex or not /// If we consider every slash token to be a regex, we could be missing cases like "1/2/3", where /// we have a series of divide operator. this list allows us to be more accurate by ruling out /// locations where a regexp cannot exist. + var noRegexTable: boolean[]; if (!noRegexTable) { noRegexTable = []; noRegexTable[SyntaxKind.Identifier] = true; @@ -4077,7 +4077,7 @@ module ts { var offset = 0; var lastTokenOrCommentEnd = 0; var lastNonTriviaToken = SyntaxKind.Unknown; - + // If we're in a string literal, then prepend: "\ // (and a newline). That way when we lex we'll think we're still in a string literal. // @@ -4098,13 +4098,14 @@ module ts { break; } + scanner.setText(text); + var result: ClassificationResult = { finalLexState: EndOfLineState.Start, entries: [] }; - scanner = createScanner(ScriptTarget.ES5, /*skipTrivia*/ false, text, /*onError*/ undefined, /*onComment*/ undefined); - + var token = SyntaxKind.Unknown; do { token = scanner.scan(); From 4493aa60ebfd2693649524d5a3cb7b1588921c70 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Tue, 23 Sep 2014 13:59:47 -0700 Subject: [PATCH 5/7] Added unit test for property access colorization. --- tests/cases/unittests/services/colorization.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/cases/unittests/services/colorization.ts b/tests/cases/unittests/services/colorization.ts index f86ee747eb4..b8a80b47ff3 100644 --- a/tests/cases/unittests/services/colorization.ts +++ b/tests/cases/unittests/services/colorization.ts @@ -168,7 +168,14 @@ describe('Colorization', function () { identifier("var")); }); - it("classifies keyword after a dot on previous line", function () { + it("classifies a property access with whitespace around the dot", function () { + test(" x .\tfoo ()", + ts.EndOfLineState.Start, + identifier("x"), + identifier("foo")); + }); + + it("classifies a keyword after a dot on previous line", function () { test("var", ts.EndOfLineState.Start, keyword("var"), From c719100f69e1d774d48046cf553b7dc6ed9d75a5 Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Tue, 23 Sep 2014 14:13:43 -0700 Subject: [PATCH 6/7] Added test cases for multiline comments in line classification. --- .../cases/unittests/services/colorization.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/cases/unittests/services/colorization.ts b/tests/cases/unittests/services/colorization.ts index b8a80b47ff3..3eb21276728 100644 --- a/tests/cases/unittests/services/colorization.ts +++ b/tests/cases/unittests/services/colorization.ts @@ -155,6 +155,27 @@ describe('Colorization', function () { finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); }); + it("correctly classifies the termination of a multiline comment", function () { + test(" */ ", + ts.EndOfLineState.InMultiLineCommentTrivia, + comment(" */"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + + it("correctly classifies the continuation of a multiline comment", function () { + test("LOREM IPSUM DOLOR ", + ts.EndOfLineState.InMultiLineCommentTrivia, + comment("LOREM IPSUM DOLOR "), + finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); + }); + + it("correctly classifies an unterminated multiline comment on a line ending in '/*/'", function () { + test(" /*/", + ts.EndOfLineState.Start, + comment("/*/"), + finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); + }); + it("classifies correctly an unterminated multiline comment with trailing space", function () { test("/* ", ts.EndOfLineState.Start, From 25170ef5dd009e85e9a2cbe6687033ecc7c3851a Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Tue, 23 Sep 2014 14:14:27 -0700 Subject: [PATCH 7/7] Changed wording for unit tests. --- tests/cases/unittests/services/colorization.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/cases/unittests/services/colorization.ts b/tests/cases/unittests/services/colorization.ts index 3eb21276728..0a5ce1f0310 100644 --- a/tests/cases/unittests/services/colorization.ts +++ b/tests/cases/unittests/services/colorization.ts @@ -105,7 +105,7 @@ describe('Colorization', function () { punctuation(";")); }); - it("classifies correctly a comment after a divide operator", function () { + it("correctly classifies a comment after a divide operator", function () { test("1 / 2 // comment", ts.EndOfLineState.Start, numberLiteral("1"), @@ -115,7 +115,7 @@ describe('Colorization', function () { comment("// comment")); }); - it("classifies correctly a literal after a divide operator", function () { + it("correctly classifies a literal after a divide operator", function () { test("1 / 2, 3 / 4", ts.EndOfLineState.Start, numberLiteral("1"), @@ -127,28 +127,28 @@ describe('Colorization', function () { operator(",")); }); - it("classifies correctly an unterminated multi-line string", function () { + it("correctly classifies an unterminated multi-line string", function () { test("'line1\\", ts.EndOfLineState.Start, stringLiteral("'line1\\"), finalEndOfLineState(ts.EndOfLineState.InSingleQuoteStringLiteral)); }); - it("classifies correctly the second line of an unterminated multi-line string", function () { + it("correctly classifies the second line of an unterminated multi-line string", function () { test("\\", ts.EndOfLineState.InDoubleQuoteStringLiteral, stringLiteral("\\"), finalEndOfLineState(ts.EndOfLineState.InDoubleQuoteStringLiteral)); }); - it("classifies correctly the last line of a multi-line string", function () { + it("correctly classifies the last line of a multi-line string", function () { test("'", ts.EndOfLineState.InSingleQuoteStringLiteral, stringLiteral("'"), finalEndOfLineState(ts.EndOfLineState.Start)); }); - it("classifies correctly an unterminated multiline comment", function () { + it("correctly classifies an unterminated multiline comment", function () { test("/*", ts.EndOfLineState.Start, comment("/*"), @@ -176,14 +176,14 @@ describe('Colorization', function () { finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); }); - it("classifies correctly an unterminated multiline comment with trailing space", function () { + it("correctly classifies an unterminated multiline comment with trailing space", function () { test("/* ", ts.EndOfLineState.Start, comment("/* "), finalEndOfLineState(ts.EndOfLineState.InMultiLineCommentTrivia)); }); - it("classifies correctly a keyword after a dot", function () { + it("correctly classifies a keyword after a dot", function () { test("a.var", ts.EndOfLineState.Start, identifier("var"));