microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2026-02-05 08:11:30 -06:00
Code Issues Packages Projects Releases 212 Wiki Activity

Added tests, fixed order of emptying templateStack, unconditionally perform template classification.

Browse Source
This commit is contained in:
Daniel Rosenwasser 2015-01-26 15:41:01 -08:00
parent 3fea0aefbe
commit ab79faef85
2 changed files with 137 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/services/services.ts
View File

@ -5663,6 +5663,11 @@ module ts {
var token = SyntaxKind.Unknown;
var lastNonTriviaToken = SyntaxKind.Unknown;
// Empty out the template stack for reuse.
while (templateStack.length > 0) {
templateStack.pop();
}
// If we're in a string literal, then prepend: "\
// (and a newline). That way when we lex we'll think we're still in a string literal.
//
@ -5682,21 +5687,15 @@ module ts {
offset = 3;
break;
case EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate:
if (syntacticClassifierAbsent) {
text = "`\n" + text;
offset = 2;
}
text = "`\n" + text;
offset = 2;
break;
case EndOfLineState.InTemplateMiddleOrTail:
if (syntacticClassifierAbsent) {
text = "}\n" + text;
offset = 2;
}
text = "}\n" + text;
offset = 2;
// fallthrough
case EndOfLineState.InTemplateSubstitutionPosition:
if (syntacticClassifierAbsent) {
templateStack = [SyntaxKind.TemplateHead];
}
templateStack.push(SyntaxKind.TemplateHead);
break;
}
@ -5728,11 +5727,6 @@ module ts {
// work well enough in practice.
var angleBracketStack = 0;
// Empty out the template stack for reuse.
while (templateStack.length > 0) {
templateStack.pop();
}
do {
token = scanner.scan();
@ -5774,17 +5768,17 @@ module ts {
token = SyntaxKind.Identifier;
}
}
else if (token === SyntaxKind.TemplateHead && syntacticClassifierAbsent) {
else if (token === SyntaxKind.TemplateHead) {
templateStack.push(token);
}
else if (token === SyntaxKind.OpenBraceToken && syntacticClassifierAbsent) {
else if (token === SyntaxKind.OpenBraceToken) {
// If we don't have anything on the template stack,
// then we aren't trying to keep track of a previously scanned template head.
if (templateStack.length > 0) {
templateStack.push(token);
}
}
else if (token === SyntaxKind.CloseBraceToken && syntacticClassifierAbsent) {
else if (token === SyntaxKind.CloseBraceToken) {
// If we don't have anything on the template stack,
// then we aren't trying to keep track of a previously scanned template head.
if (templateStack.length > 0) {
@ -5821,7 +5815,7 @@ module ts {
var start = scanner.getTokenPos();
var end = scanner.getTextPos();
addResult(end - start, classFromKind(token, syntacticClassifierAbsent));
addResult(end - start, classFromKind(token));
if (end >= text.length) {
if (token === SyntaxKind.StringLiteral) {
@ -5850,7 +5844,7 @@ module ts {
result.finalLexState = EndOfLineState.InMultiLineCommentTrivia;
}
}
else if (isTemplateLiteralKind(token) && syntacticClassifierAbsent) {
else if (isTemplateLiteralKind(token)) {
if (scanner.isUnterminated()) {
if (token === SyntaxKind.TemplateTail) {
result.finalLexState = EndOfLineState.InTemplateMiddleOrTail;
@ -5943,7 +5937,7 @@ module ts {
return token >= SyntaxKind.FirstKeyword && token <= SyntaxKind.LastKeyword;
}
function classFromKind(token: SyntaxKind, syntacticClassifierAbsent?: boolean) {
function classFromKind(token: SyntaxKind) {
if (isKeyword(token)) {
return TokenClass.Keyword;
}
@ -5969,9 +5963,8 @@ module ts {
return TokenClass.Whitespace;
case SyntaxKind.Identifier:
default:
// Only give a classification if nothing will more accurately classify.
if (syntacticClassifierAbsent && isTemplateLiteralKind(token)) {
return TokenClass.StringLiteral; // should make a TemplateLiteral
if (isTemplateLiteralKind(token)) {
return TokenClass.StringLiteral; // maybe make a TemplateLiteral
}
return TokenClass.Identifier;
}

130
tests/cases/unittests/services/colorization.ts
View File

@ -4,6 +4,7 @@
interface ClassificationEntry {
value: any;
classification: ts.TokenClass;
position?: number;
}
describe('Colorization', function () {
@ -23,16 +24,23 @@ describe('Colorization', function () {
return undefined;
}
function punctuation(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Punctuation }; }
function keyword(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Keyword }; }
function operator(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Operator }; }
function comment(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Comment }; }
function whitespace(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Whitespace }; }
function identifier(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Identifier }; }
function numberLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.NumberLiteral }; }
function stringLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.StringLiteral }; }
function regExpLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.RegExpLiteral }; }
function finalEndOfLineState(value: number): ClassificationEntry { return { value: value, classification: <ts.TokenClass>undefined }; }
function punctuation(text: string, position?: number) { return createClassification(text, ts.TokenClass.Punctuation, position); }
function keyword(text: string, position?: number) { return createClassification(text, ts.TokenClass.Keyword, position); }
function operator(text: string, position?: number) { return createClassification(text, ts.TokenClass.Operator, position); }
function comment(text: string, position?: number) { return createClassification(text, ts.TokenClass.Comment, position); }
function whitespace(text: string, position?: number) { return createClassification(text, ts.TokenClass.Whitespace, position); }
function identifier(text: string, position?: number) { return createClassification(text, ts.TokenClass.Identifier, position); }
function numberLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.NumberLiteral, position); }
function stringLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.StringLiteral, position); }
function regExpLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.RegExpLiteral, position); }
function finalEndOfLineState(value: number): ClassificationEntry { return { value: value, classification: undefined, position: 0 }; }
function createClassification(text: string, tokenClass: ts.TokenClass, position?: number): ClassificationEntry {
return {
value: text,
classification: tokenClass,
position: position,
};
}
function testLexicalClassification(text: string, initialEndOfLineState: ts.EndOfLineState, ...expectedEntries: ClassificationEntry[]): void {
var result = classifier.getClassificationsForLine(text, initialEndOfLineState);
@ -44,7 +52,7 @@ describe('Colorization', function () {
assert.equal(result.finalLexState, expectedEntry.value, "final endOfLineState does not match expected.");
}
else {
var actualEntryPosition = text.indexOf(expectedEntry.value);
var actualEntryPosition = expectedEntry.position !== undefined ? expectedEntry.position : text.indexOf(expectedEntry.value);
assert(actualEntryPosition >= 0, "token: '" + expectedEntry.value + "' does not exit in text: '" + text + "'.");
var actualEntry = getEntryAtPosistion(result, actualEntryPosition);
@ -254,6 +262,106 @@ describe('Colorization', function () {
finalEndOfLineState(ts.EndOfLineState.Start));
});
it("classifies a single line no substitution template string correctly", () => {
testLexicalClassification("`number number public string`",
ts.EndOfLineState.Start,
stringLiteral("`number number public string`"),
finalEndOfLineState(ts.EndOfLineState.Start));
});
it("classifies substitution parts of a template string correctly", () => {
testLexicalClassification("`number '${ 1 + 1 }' string '${ 'hello' }'`",
ts.EndOfLineState.Start,
stringLiteral("`number '${"),
numberLiteral("1"),
operator("+"),
numberLiteral("1"),
stringLiteral("}' string '${"),
stringLiteral("'hello'"),
stringLiteral("}'`"),
finalEndOfLineState(ts.EndOfLineState.Start));
});
it("classifies an unterminated no substitution template string correctly", () => {
testLexicalClassification("`hello world",
ts.EndOfLineState.Start,
stringLiteral("`hello world"),
finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate));
});
it("classifies the entire line of an unterminated multiline no-substitution/head template", () => {
testLexicalClassification("...",
ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate,
stringLiteral("..."),
finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate));
});
it("classifies the entire line of an unterminated multiline template middle/end",() => {
testLexicalClassification("...",
ts.EndOfLineState.InTemplateMiddleOrTail,
stringLiteral("..."),
finalEndOfLineState(ts.EndOfLineState.InTemplateMiddleOrTail));
});
it("classifies a termination of a multiline template head", () => {
testLexicalClassification("...${",
ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate,
stringLiteral("...${"),
finalEndOfLineState(ts.EndOfLineState.InTemplateSubstitutionPosition));
});
it("classifies the termination of a multiline no substitution template", () => {
testLexicalClassification("...`",
ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate,
stringLiteral("...`"),
finalEndOfLineState(ts.EndOfLineState.Start));
});
it("classifies the substitution parts and middle/tail of a multiline template string", () => {
testLexicalClassification("${ 1 + 1 }...`",
ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate,
stringLiteral("${"),
numberLiteral("1"),
operator("+"),
numberLiteral("1"),
stringLiteral("}...`"),
finalEndOfLineState(ts.EndOfLineState.Start));
});
it("classifies a template middle and propagates the end of line state",() => {
testLexicalClassification("${ 1 + 1 }...`",
ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate,
stringLiteral("${"),
numberLiteral("1"),
operator("+"),
numberLiteral("1"),
stringLiteral("}...`"),
finalEndOfLineState(ts.EndOfLineState.Start));
});
it("classifies substitution expressions with curly braces appropriately", () => {
var pos = 0;
var lastLength = 0;
testLexicalClassification("...${ () => { } } ${ { x: `1` } }...`",
ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate,
stringLiteral(track("...${"), pos),
punctuation(track(" ", "("), pos),
punctuation(track(")"), pos),
punctuation(track(" ", "=>"), pos),
punctuation(track(" ", "{"), pos),
punctuation(track(" ", "}"), pos),
stringLiteral(track(" ", "} ${"), pos),
punctuation(track(" ", "{"), pos),
identifier(track(" ", "x"), pos),
punctuation(track(":"), pos),
stringLiteral(track(" ", "`1`"), pos),
punctuation(track(" ", "}"), pos),
stringLiteral(track(" ", "}...`"), pos),
finalEndOfLineState(ts.EndOfLineState.Start));
// Adjusts 'pos' by accounting for the length of each portion of the string,
// but only return the last given string
function track(...vals: string[]): string {
for (var i = 0, n = vals.length; i < n; i++) {
pos += lastLength;
lastLength = vals[i].length;
}
return ts.lastOrUndefined(vals);
}
});
it("classifies partially written generics correctly.", function () {
testLexicalClassification("Foo<number",
ts.EndOfLineState.Start,