From ab79faef85568a9944709c0940c6f484aaa5187c Mon Sep 17 00:00:00 2001 From: Daniel Rosenwasser Date: Mon, 26 Jan 2015 15:41:01 -0800 Subject: [PATCH] Added tests, fixed order of emptying templateStack, unconditionally perform template classification. --- src/services/services.ts | 43 +++--- .../cases/unittests/services/colorization.ts | 130 ++++++++++++++++-- 2 files changed, 137 insertions(+), 36 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index 172fb4ab00e..673748d6969 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -5663,6 +5663,11 @@ module ts { var token = SyntaxKind.Unknown; var lastNonTriviaToken = SyntaxKind.Unknown; + // Empty out the template stack for reuse. + while (templateStack.length > 0) { + templateStack.pop(); + } + // If we're in a string literal, then prepend: "\ // (and a newline). That way when we lex we'll think we're still in a string literal. // @@ -5682,21 +5687,15 @@ module ts { offset = 3; break; case EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate: - if (syntacticClassifierAbsent) { - text = "`\n" + text; - offset = 2; - } + text = "`\n" + text; + offset = 2; break; case EndOfLineState.InTemplateMiddleOrTail: - if (syntacticClassifierAbsent) { - text = "}\n" + text; - offset = 2; - } + text = "}\n" + text; + offset = 2; // fallthrough case EndOfLineState.InTemplateSubstitutionPosition: - if (syntacticClassifierAbsent) { - templateStack = [SyntaxKind.TemplateHead]; - } + templateStack.push(SyntaxKind.TemplateHead); break; } @@ -5728,11 +5727,6 @@ module ts { // work well enough in practice. var angleBracketStack = 0; - // Empty out the template stack for reuse. - while (templateStack.length > 0) { - templateStack.pop(); - } - do { token = scanner.scan(); @@ -5774,17 +5768,17 @@ module ts { token = SyntaxKind.Identifier; } } - else if (token === SyntaxKind.TemplateHead && syntacticClassifierAbsent) { + else if (token === SyntaxKind.TemplateHead) { templateStack.push(token); } - else if (token === SyntaxKind.OpenBraceToken && syntacticClassifierAbsent) { + else if (token === SyntaxKind.OpenBraceToken) { // If we don't have anything on the template stack, // then we aren't trying to keep track of a previously scanned template head. if (templateStack.length > 0) { templateStack.push(token); } } - else if (token === SyntaxKind.CloseBraceToken && syntacticClassifierAbsent) { + else if (token === SyntaxKind.CloseBraceToken) { // If we don't have anything on the template stack, // then we aren't trying to keep track of a previously scanned template head. if (templateStack.length > 0) { @@ -5821,7 +5815,7 @@ module ts { var start = scanner.getTokenPos(); var end = scanner.getTextPos(); - addResult(end - start, classFromKind(token, syntacticClassifierAbsent)); + addResult(end - start, classFromKind(token)); if (end >= text.length) { if (token === SyntaxKind.StringLiteral) { @@ -5850,7 +5844,7 @@ module ts { result.finalLexState = EndOfLineState.InMultiLineCommentTrivia; } } - else if (isTemplateLiteralKind(token) && syntacticClassifierAbsent) { + else if (isTemplateLiteralKind(token)) { if (scanner.isUnterminated()) { if (token === SyntaxKind.TemplateTail) { result.finalLexState = EndOfLineState.InTemplateMiddleOrTail; @@ -5943,7 +5937,7 @@ module ts { return token >= SyntaxKind.FirstKeyword && token <= SyntaxKind.LastKeyword; } - function classFromKind(token: SyntaxKind, syntacticClassifierAbsent?: boolean) { + function classFromKind(token: SyntaxKind) { if (isKeyword(token)) { return TokenClass.Keyword; } @@ -5969,9 +5963,8 @@ module ts { return TokenClass.Whitespace; case SyntaxKind.Identifier: default: - // Only give a classification if nothing will more accurately classify. - if (syntacticClassifierAbsent && isTemplateLiteralKind(token)) { - return TokenClass.StringLiteral; // should make a TemplateLiteral + if (isTemplateLiteralKind(token)) { + return TokenClass.StringLiteral; // maybe make a TemplateLiteral } return TokenClass.Identifier; } diff --git a/tests/cases/unittests/services/colorization.ts b/tests/cases/unittests/services/colorization.ts index 01149764305..c834f85806d 100644 --- a/tests/cases/unittests/services/colorization.ts +++ b/tests/cases/unittests/services/colorization.ts @@ -4,6 +4,7 @@ interface ClassificationEntry { value: any; classification: ts.TokenClass; + position?: number; } describe('Colorization', function () { @@ -23,16 +24,23 @@ describe('Colorization', function () { return undefined; } - function punctuation(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Punctuation }; } - function keyword(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Keyword }; } - function operator(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Operator }; } - function comment(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Comment }; } - function whitespace(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Whitespace }; } - function identifier(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Identifier }; } - function numberLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.NumberLiteral }; } - function stringLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.StringLiteral }; } - function regExpLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.RegExpLiteral }; } - function finalEndOfLineState(value: number): ClassificationEntry { return { value: value, classification: undefined }; } + function punctuation(text: string, position?: number) { return createClassification(text, ts.TokenClass.Punctuation, position); } + function keyword(text: string, position?: number) { return createClassification(text, ts.TokenClass.Keyword, position); } + function operator(text: string, position?: number) { return createClassification(text, ts.TokenClass.Operator, position); } + function comment(text: string, position?: number) { return createClassification(text, ts.TokenClass.Comment, position); } + function whitespace(text: string, position?: number) { return createClassification(text, ts.TokenClass.Whitespace, position); } + function identifier(text: string, position?: number) { return createClassification(text, ts.TokenClass.Identifier, position); } + function numberLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.NumberLiteral, position); } + function stringLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.StringLiteral, position); } + function regExpLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.RegExpLiteral, position); } + function finalEndOfLineState(value: number): ClassificationEntry { return { value: value, classification: undefined, position: 0 }; } + function createClassification(text: string, tokenClass: ts.TokenClass, position?: number): ClassificationEntry { + return { + value: text, + classification: tokenClass, + position: position, + }; + } function testLexicalClassification(text: string, initialEndOfLineState: ts.EndOfLineState, ...expectedEntries: ClassificationEntry[]): void { var result = classifier.getClassificationsForLine(text, initialEndOfLineState); @@ -44,7 +52,7 @@ describe('Colorization', function () { assert.equal(result.finalLexState, expectedEntry.value, "final endOfLineState does not match expected."); } else { - var actualEntryPosition = text.indexOf(expectedEntry.value); + var actualEntryPosition = expectedEntry.position !== undefined ? expectedEntry.position : text.indexOf(expectedEntry.value); assert(actualEntryPosition >= 0, "token: '" + expectedEntry.value + "' does not exit in text: '" + text + "'."); var actualEntry = getEntryAtPosistion(result, actualEntryPosition); @@ -254,6 +262,106 @@ describe('Colorization', function () { finalEndOfLineState(ts.EndOfLineState.Start)); }); + it("classifies a single line no substitution template string correctly", () => { + testLexicalClassification("`number number public string`", + ts.EndOfLineState.Start, + stringLiteral("`number number public string`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies substitution parts of a template string correctly", () => { + testLexicalClassification("`number '${ 1 + 1 }' string '${ 'hello' }'`", + ts.EndOfLineState.Start, + stringLiteral("`number '${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}' string '${"), + stringLiteral("'hello'"), + stringLiteral("}'`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies an unterminated no substitution template string correctly", () => { + testLexicalClassification("`hello world", + ts.EndOfLineState.Start, + stringLiteral("`hello world"), + finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); + }); + it("classifies the entire line of an unterminated multiline no-substitution/head template", () => { + testLexicalClassification("...", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("..."), + finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate)); + }); + it("classifies the entire line of an unterminated multiline template middle/end",() => { + testLexicalClassification("...", + ts.EndOfLineState.InTemplateMiddleOrTail, + stringLiteral("..."), + finalEndOfLineState(ts.EndOfLineState.InTemplateMiddleOrTail)); + }); + it("classifies a termination of a multiline template head", () => { + testLexicalClassification("...${", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("...${"), + finalEndOfLineState(ts.EndOfLineState.InTemplateSubstitutionPosition)); + }); + it("classifies the termination of a multiline no substitution template", () => { + testLexicalClassification("...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies the substitution parts and middle/tail of a multiline template string", () => { + testLexicalClassification("${ 1 + 1 }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies a template middle and propagates the end of line state",() => { + testLexicalClassification("${ 1 + 1 }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral("${"), + numberLiteral("1"), + operator("+"), + numberLiteral("1"), + stringLiteral("}...`"), + finalEndOfLineState(ts.EndOfLineState.Start)); + }); + it("classifies substitution expressions with curly braces appropriately", () => { + var pos = 0; + var lastLength = 0; + + testLexicalClassification("...${ () => { } } ${ { x: `1` } }...`", + ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate, + stringLiteral(track("...${"), pos), + punctuation(track(" ", "("), pos), + punctuation(track(")"), pos), + punctuation(track(" ", "=>"), pos), + punctuation(track(" ", "{"), pos), + punctuation(track(" ", "}"), pos), + stringLiteral(track(" ", "} ${"), pos), + punctuation(track(" ", "{"), pos), + identifier(track(" ", "x"), pos), + punctuation(track(":"), pos), + stringLiteral(track(" ", "`1`"), pos), + punctuation(track(" ", "}"), pos), + stringLiteral(track(" ", "}...`"), pos), + finalEndOfLineState(ts.EndOfLineState.Start)); + + // Adjusts 'pos' by accounting for the length of each portion of the string, + // but only return the last given string + function track(...vals: string[]): string { + for (var i = 0, n = vals.length; i < n; i++) { + pos += lastLength; + lastLength = vals[i].length; + } + return ts.lastOrUndefined(vals); + } + }); + it("classifies partially written generics correctly.", function () { testLexicalClassification("Foo