diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts index 921a34d3895..61cb6628f25 100644 --- a/src/compiler/parser.ts +++ b/src/compiler/parser.ts @@ -3815,7 +3815,7 @@ module ts { : undefined); } - scanner = createScanner(languageVersion, sourceText, scanError, onComment); + scanner = createScanner(languageVersion, /*skipTrivia*/ true, sourceText, scanError, onComment); var rootNodeFlags: NodeFlags = 0; if (fileExtensionIs(filename, ".d.ts")) { rootNodeFlags = NodeFlags.DeclarationFile; diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index 59c7913e24a..f627bee6c51 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -460,7 +460,7 @@ module ts { ch > CharacterCodes.maxAsciiCharacter && isUnicodeIdentifierPart(ch, languageVersion); } - export function createScanner(languageVersion: ScriptTarget, text?: string, onError?: ErrorCallback, onComment?: CommentCallback): Scanner { + export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback, onComment?: CommentCallback): Scanner { var pos: number; // Current position (end position of text of current token) var len: number; // Length of text var startPos: number; // Start position of whitespace before current token @@ -694,12 +694,34 @@ module ts { case CharacterCodes.lineFeed: case CharacterCodes.carriageReturn: precedingLineBreak = true; + if (skipTrivia) { + pos++; + continue; + } + else { + if (ch === CharacterCodes.carriageReturn && pos + 1 < len && text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) { + // consume both CR and LF + pos += 2; + } + else { + pos++; + } + return token = SyntaxKind.NewLineTrivia; + } case CharacterCodes.tab: case CharacterCodes.verticalTab: case CharacterCodes.formFeed: case CharacterCodes.space: - pos++; - continue; + if (skipTrivia) { + pos++; + continue; + } + else { + while (pos < len && isWhiteSpace(text.charCodeAt(pos))) { + pos++; + } + return token = SyntaxKind.WhitespaceTrivia; + } case CharacterCodes.exclamation: if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { if (text.charCodeAt(pos + 2) === CharacterCodes.equals) { @@ -776,7 +798,13 @@ module ts { if (onComment) { onComment(tokenPos, pos); } - continue; + + if (skipTrivia) { + continue; + } + else { + return token = SyntaxKind.SingleLineCommentTrivia; + } } // Multi-line comment if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) { @@ -806,7 +834,12 @@ module ts { onComment(tokenPos, pos); } - continue; + if (skipTrivia) { + continue; + } + else { + return token = SyntaxKind.MultiLineCommentTrivia; + } } if (text.charCodeAt(pos + 1) === CharacterCodes.equals) { diff --git a/src/compiler/types.ts b/src/compiler/types.ts index 82fac6b8913..e1116150e34 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -12,6 +12,10 @@ module ts { export enum SyntaxKind { Unknown, EndOfFileToken, + SingleLineCommentTrivia, + MultiLineCommentTrivia, + NewLineTrivia, + WhitespaceTrivia, // Literals NumericLiteral, StringLiteral, diff --git a/src/services/services.ts b/src/services/services.ts index ac7d761a612..4a64aee191c 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -75,7 +75,7 @@ module ts { update(scriptSnapshot: TypeScript.IScriptSnapshot, version: string, isOpen: boolean, textChangeRange: TypeScript.TextChangeRange): SourceFile; } - var scanner: Scanner = createScanner(ScriptTarget.ES5); + var scanner: Scanner = createScanner(ScriptTarget.ES5, /*skipTrivia*/ true); var emptyArray: any[] = []; @@ -4070,7 +4070,7 @@ module ts { entries: [] }; - scanner = createScanner(ScriptTarget.ES5, text, onError, processComment); + scanner = createScanner(ScriptTarget.ES5, /*skipTrivia*/ true, text, onError, processComment); var token = SyntaxKind.Unknown; do {