microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2026-02-14 10:55:15 -06:00
Code Issues Packages Projects Releases 212 Wiki Activity

Use the 'skipTrivia' scanner flag for lexical classification.

Browse Source
This commit is contained in:
Daniel Rosenwasser 2014-09-23 12:25:57 -07:00
parent b2372c9e7b
commit 292cfc8721
2 changed files with 16 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/compiler/types.ts
View File

@ -228,7 +228,9 @@ module ts {
FirstPunctuation = OpenBraceToken,
LastPunctuation = CaretEqualsToken,
FirstToken = EndOfFileToken,
LastToken = StringKeyword
LastToken = StringKeyword,
FirstTriviaToken = SingleLineCommentTrivia,
LastTriviaToken = WhitespaceTrivia
}
export enum NodeFlags {

38
src/services/services.ts
View File

@ -4076,7 +4076,7 @@ module ts {
function getClassificationsForLine(text: string, lexState: EndOfLineState): ClassificationResult {
var offset = 0;
var lastTokenOrCommentEnd = 0;
var lastToken = SyntaxKind.Unknown;
var lastNonTriviaToken = SyntaxKind.Unknown;
var inUnterminatedMultiLineComment = false;
// If we're in a string literal, then prepend: "\
@ -4104,22 +4104,25 @@ module ts {
entries: []
};
scanner = createScanner(ScriptTarget.ES5, /*skipTrivia*/ true, text, onError, processComment);
scanner = createScanner(ScriptTarget.ES5, /*skipTrivia*/ false, text, onError, /*onComment*/ undefined);
var token = SyntaxKind.Unknown;
do {
token = scanner.scan();
if ((token === SyntaxKind.SlashToken || token === SyntaxKind.SlashEqualsToken) && !noRegexTable[lastToken]) {
if ((token === SyntaxKind.SlashToken || token === SyntaxKind.SlashEqualsToken) && !noRegexTable[lastNonTriviaToken]) {
if (scanner.reScanSlashToken() === SyntaxKind.RegularExpressionLiteral) {
token = SyntaxKind.RegularExpressionLiteral;
}
}
else if (lastToken === SyntaxKind.DotToken) {
else if (lastNonTriviaToken === SyntaxKind.DotToken) {
token = SyntaxKind.Identifier;
}
lastToken = token;
// Only recall the token if it was *not* trivia.
if (!(SyntaxKind.FirstTriviaToken <= token && token <= SyntaxKind.LastTriviaToken)) {
lastNonTriviaToken = token;
}
processToken();
}
@ -4132,21 +4135,10 @@ module ts {
inUnterminatedMultiLineComment = message.key === Diagnostics.Asterisk_Slash_expected.key;
}
function processComment(start: number, end: number) {
// add Leading white spaces
addLeadingWhiteSpace(start, end);
// add the comment
addResult(end - start, TokenClass.Comment);
}
function processToken(): void {
var start = scanner.getTokenPos();
var end = scanner.getTextPos();
// add Leading white spaces
addLeadingWhiteSpace(start, end);
// add the token
addResult(end - start, classFromKind(token));
@ -4167,15 +4159,6 @@ module ts {
}
}
function addLeadingWhiteSpace(start: number, end: number): void {
if (start > lastTokenOrCommentEnd) {
addResult(start - lastTokenOrCommentEnd, TokenClass.Whitespace);
}
// Remember the end of the last token
lastTokenOrCommentEnd = end;
}
function addResult(length: number, classification: TokenClass): void {
if (length > 0) {
// If this is the first classification we're adding to the list, then remove any
@ -4268,6 +4251,11 @@ module ts {
return TokenClass.StringLiteral;
case SyntaxKind.RegularExpressionLiteral:
return TokenClass.RegExpLiteral;
case SyntaxKind.MultiLineCommentTrivia:
case SyntaxKind.SingleLineCommentTrivia:
return TokenClass.Comment;
case SyntaxKind.WhitespaceTrivia:
return TokenClass.Whitespace;
case SyntaxKind.Identifier:
default:
return TokenClass.Identifier;