microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2026-05-26 10:43:51 -05:00
Code Issues Packages Projects Releases 212 Wiki Activity

Move to a syntax model where we only have leading trivia.

Browse Source
This commit is contained in:
Cyrus Najmabadi
2014-11-08 23:30:44 -08:00
parent f02ba7f212
commit 37e9c2db89
13 changed files with 79 additions and 456 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/services/formatting/formatter.ts
View File

@@ -96,11 +96,6 @@ module TypeScript.Services.Formatting {
}
this.previousTokenParent = this.parent().clone(this.indentationNodeContextPool());
position += width(token);
// Extract any trailing comments
if (token.trailingTriviaWidth() !== 0) {
this.processTrivia(token.trailingTrivia(), position);
}
}
private processTrivia(triviaList: ISyntaxTriviaList, fullStart: number) {

2
src/services/formatting/formattingContext.ts
View File

@@ -109,7 +109,7 @@ module TypeScript.Services.Formatting {
var block = <BlockSyntax>node.node();
// Now check if they are on the same line
return this.snapshot.getLineNumberFromPosition(end(block.openBraceToken)) ===
return this.snapshot.getLineNumberFromPosition(fullEnd(block.openBraceToken)) ===
this.snapshot.getLineNumberFromPosition(start(block.closeBraceToken));
}
}

4
src/services/formatting/formattingManager.ts
View File

@@ -46,7 +46,7 @@ module TypeScript.Services.Formatting {
// Find the outer most parent that this semicolon terminates
var current: ISyntaxElement = semicolonPositionedToken;
while (current.parent !== null &&
end(current.parent) === end(semicolonPositionedToken) &&
fullEnd(current.parent) === fullEnd(semicolonPositionedToken) &&
current.parent.kind !== SyntaxKind.List) {
current = current.parent;
}
@@ -69,7 +69,7 @@ module TypeScript.Services.Formatting {
// Find the outer most parent that this closing brace terminates
var current: ISyntaxElement = closeBracePositionedToken;
while (current.parent !== null &&
end(current.parent) === end(closeBracePositionedToken) &&
fullEnd(current.parent) === fullEnd(closeBracePositionedToken) &&
current.parent.kind !== SyntaxKind.List) {
current = current.parent;
}

7
src/services/formatting/indentationTrackingWalker.ts
View File

@@ -90,8 +90,11 @@ module TypeScript.Services.Formatting {
this.visitTokenInSpan(token);
// Only track new lines on tokens within the range. Make sure to check that the last trivia is a newline, and not just one of the trivia
var trivia = token.trailingTrivia();
this._lastTriviaWasNewLine = trivia.hasNewLine() && trivia.syntaxTriviaAt(trivia.count() - 1).kind == SyntaxKind.NewLineTrivia;
var _nextToken = nextToken(token);
if (_nextToken && _nextToken.hasLeadingTrivia()) {
var trivia = _nextToken.leadingTrivia();
this._lastTriviaWasNewLine = trivia.hasNewLine();
}
}
// Update the position

105
src/services/indentation.ts
View File

@@ -1,110 +1,5 @@
module TypeScript.Indentation {
export function columnForEndOfTokenAtPosition(syntaxTree: SyntaxTree, position: number, options: FormattingOptions): number {
var token = findToken(syntaxTree.sourceUnit(), position);
return columnForStartOfTokenAtPosition(syntaxTree, position, options) + width(token);
}
export function columnForStartOfTokenAtPosition(syntaxTree: SyntaxTree, position: number, options: FormattingOptions): number {
var token = findToken(syntaxTree.sourceUnit(), position);
// Walk backward from this token until we find the first token in the line. For each token
// we see (that is not the first tokem in line), push the entirety of the text into the text
// array. Then, for the first token, add its text (without its leading trivia) to the text
// array. i.e. if we have:
//
// var foo = a => bar();
//
// And we want the column for the start of 'bar', then we'll add the underlinded portions to
// the text array:
//
// var foo = a => bar();
// _
// __
// __
// ____
// ____
var firstTokenInLine = Syntax.firstTokenInLineContainingPosition(syntaxTree, token.fullStart());
var leadingTextInReverse: string[] = [];
var current = token;
while (current !== firstTokenInLine) {
current = previousToken(current);
if (current === firstTokenInLine) {
// We're at the first token in teh line.
// We don't want the leading trivia for this token. That will be taken care of in
// columnForFirstNonWhitespaceCharacterInLine. So just push the trailing trivia
// and then the token text.
leadingTextInReverse.push(current.trailingTrivia().fullText());
leadingTextInReverse.push(current.text());
}
else {
// We're at an intermediate token on the line. Just push all its text into the array.
leadingTextInReverse.push(current.fullText());
}
}
// Now, add all trivia to the start of the line on the first token in the list.
collectLeadingTriviaTextToStartOfLine(firstTokenInLine, leadingTextInReverse);
return columnForLeadingTextInReverse(leadingTextInReverse, options);
}
export function columnForStartOfFirstTokenInLineContainingPosition(syntaxTree: SyntaxTree, position: number, options: FormattingOptions): number {
// Walk backward through the tokens until we find the first one on the line.
var firstTokenInLine = Syntax.firstTokenInLineContainingPosition(syntaxTree, position);
var leadingTextInReverse: string[] = [];
// Now, add all trivia to the start of the line on the first token in the list.
collectLeadingTriviaTextToStartOfLine(firstTokenInLine, leadingTextInReverse);
return columnForLeadingTextInReverse(leadingTextInReverse, options);
}
// Collect all the trivia that precedes this token. Stopping when we hit a newline trivia
// or a multiline comment that spans multiple lines. This is meant to be called on the first
// token in a line.
function collectLeadingTriviaTextToStartOfLine(firstTokenInLine: ISyntaxToken,
leadingTextInReverse: string[]) {
var leadingTrivia = firstTokenInLine.leadingTrivia();
for (var i = leadingTrivia.count() - 1; i >= 0; i--) {
var trivia = leadingTrivia.syntaxTriviaAt(i);
if (trivia.kind === SyntaxKind.NewLineTrivia) {
break;
}
if (trivia.kind === SyntaxKind.MultiLineCommentTrivia) {
var lineSegments = Syntax.splitMultiLineCommentTriviaIntoMultipleLines(trivia);
leadingTextInReverse.push(ArrayUtilities.last(lineSegments));
if (lineSegments.length > 0) {
// This multiline comment actually spanned multiple lines. So we're done.
break;
}
// It was only on a single line, so keep on going.
}
leadingTextInReverse.push(trivia.fullText());
}
}
function columnForLeadingTextInReverse(leadingTextInReverse: string[],
options: FormattingOptions): number {
var column = 0;
// walk backwards. This means we're actually walking forward from column 0 to the start of
// the token.
for (var i = leadingTextInReverse.length - 1; i >= 0; i--) {
var text = leadingTextInReverse[i];
column = columnForPositionInStringWorker(text, text.length, column, options);
}
return column;
}
// Returns the column that this input string ends at (assuming it starts at column 0).
export function columnForPositionInString(input: string, position: number, options: FormattingOptions): number {
return columnForPositionInStringWorker(input, position, 0, options);

31
src/services/syntax/parser.ts
View File

@@ -332,16 +332,13 @@ module TypeScript.Parser {
addTriviaTo(skippedToken.leadingTrivia(source.text), array);
// now, add the text of the token as skipped text to the trivia array.
var trimmedToken = Syntax.withTrailingTrivia(Syntax.withLeadingTrivia(skippedToken, Syntax.emptyTriviaList, source.text), Syntax.emptyTriviaList, source.text);
var trimmedToken = Syntax.withLeadingTrivia(skippedToken, Syntax.emptyTriviaList, source.text);
// Because we removed the leading trivia from the skipped token, the full start of the
// trimmed token is the start of the skipped token.
trimmedToken.setFullStart(start(skippedToken, source.text));
array.push(Syntax.skippedTokenTrivia(trimmedToken, source.text));
// Finally, add the trailing trivia of the skipped token to the trivia array.
addTriviaTo(skippedToken.trailingTrivia(source.text), array);
}
function addTriviaTo(list: ISyntaxTriviaList, array: ISyntaxTrivia[]): void {
@@ -446,19 +443,21 @@ module TypeScript.Parser {
}
function previousTokenHasTrailingNewLine(token: ISyntaxToken): boolean {
var tokenFullStart = token.fullStart();
if (tokenFullStart === 0) {
// First token in the document. Thus it has no 'previous' token, and there is
// no preceding newline.
return false;
}
return token.hasLeadingNewLine();
// If our previous token ended with a newline, then *by definition* we must have started
// at the beginning of a line.
var lineNumber = source.text.lineMap().getLineNumberFromPosition(tokenFullStart);
var lineStart = source.text.lineMap().getLineStartPosition(lineNumber);
//var tokenFullStart = token.fullStart();
//if (tokenFullStart === 0) {
// // First token in the document. Thus it has no 'previous' token, and there is
// // no preceding newline.
// return false;
//}
return lineStart == tokenFullStart;
//// If our previous token ended with a newline, then *by definition* we must have started
//// at the beginning of a line.
//var lineNumber = source.text.lineMap().getLineNumberFromPosition(tokenFullStart);
//var lineStart = source.text.lineMap().getLineStartPosition(lineNumber);
//return lineStart == tokenFullStart;
}
function canEatAutomaticSemicolon(allowWithoutNewLine: boolean): boolean {
@@ -2717,7 +2716,7 @@ module TypeScript.Parser {
// and report a better error message.
if (inObjectCreation && currentToken().kind === SyntaxKind.CloseBracketToken) {
var errorStart = start(openBracketToken, source.text);
var errorEnd = end(currentToken(), source.text);
var errorEnd = fullEnd(currentToken());
var diagnostic = new Diagnostic(fileName, source.text.lineMap(), errorStart, errorEnd - errorStart,
DiagnosticCode.new_T_cannot_be_used_to_create_an_array_Use_new_Array_T_instead, undefined);
addDiagnostic(diagnostic);

129
src/services/syntax/scanner.ts
View File

@@ -60,8 +60,7 @@ module TypeScript.Scanner {
// This gives us 23bit for width (or 8MB of width which should be enough for any codebase).
enum ScannerConstants {
LargeTokenFullWidthShift = 6,
LargeTokenLeadingTriviaShift = 3,
LargeTokenFullWidthShift = 3,
WhitespaceTrivia = 0x01, // 00000001
NewlineTrivia = 0x02, // 00000010
@@ -72,8 +71,8 @@ module TypeScript.Scanner {
IsVariableWidthMask = 0x80, // 10000000
}
function largeTokenPackData(fullWidth: number, leadingTriviaInfo: number, trailingTriviaInfo: number) {
return (fullWidth << ScannerConstants.LargeTokenFullWidthShift) | (leadingTriviaInfo << ScannerConstants.LargeTokenLeadingTriviaShift) | trailingTriviaInfo;
function largeTokenPackData(fullWidth: number, leadingTriviaInfo: number) {
return (fullWidth << ScannerConstants.LargeTokenFullWidthShift) | leadingTriviaInfo;
}
function largeTokenUnpackFullWidth(packedFullWidthAndInfo: number): number {
@@ -81,10 +80,6 @@ module TypeScript.Scanner {
}
function largeTokenUnpackLeadingTriviaInfo(packedFullWidthAndInfo: number): number {
return (packedFullWidthAndInfo >> ScannerConstants.LargeTokenLeadingTriviaShift) & ScannerConstants.TriviaMask;
}
function largeTokenUnpackTrailingTriviaInfo(packedFullWidthAndInfo: number): number {
return packedFullWidthAndInfo & ScannerConstants.TriviaMask;
}
@@ -92,20 +87,20 @@ module TypeScript.Scanner {
return largeTokenUnpackLeadingTriviaInfo(packed) !== 0;
}
function largeTokenUnpackHasTrailingTrivia(packed: number): boolean {
return largeTokenUnpackTrailingTriviaInfo(packed) !== 0;
}
function hasComment(info: number) {
return (info & ScannerConstants.CommentTrivia) !== 0;
}
function largeTokenUnpackHasLeadingComment(packed: number): boolean {
return hasComment(largeTokenUnpackLeadingTriviaInfo(packed));
function hasNewLine(info: number) {
return (info & ScannerConstants.NewlineTrivia) !== 0;
}
function largeTokenUnpackHasTrailingComment(packed: number): boolean {
return hasComment(largeTokenUnpackTrailingTriviaInfo(packed));
function largeTokenUnpackHasLeadingNewLine(packed: number): boolean {
return hasNewLine(largeTokenUnpackLeadingTriviaInfo(packed));
}
function largeTokenUnpackHasLeadingComment(packed: number): boolean {
return hasComment(largeTokenUnpackLeadingTriviaInfo(packed));
}
var isKeywordStartCharacter: number[] = ArrayUtilities.createArray<number>(CharacterCodes.maxAsciiCharacter, 0);
@@ -156,7 +151,7 @@ module TypeScript.Scanner {
}
}
var lastTokenInfo = { leadingTriviaWidth: -1, width: -1 };
var lastTokenInfo = { leadingTriviaWidth: -1 };
var lastTokenInfoTokenID: number = -1;
var triviaScanner = createScannerInternal(ts.ScriptTarget.Latest, SimpleText.fromString(""), () => { });
@@ -180,15 +175,7 @@ module TypeScript.Scanner {
return Syntax.emptyTriviaList;
}
return triviaScanner.scanTrivia(token, text, /*isTrailing:*/ false);
}
function trailingTrivia(token: IScannerToken, text: ISimpleText): ISyntaxTriviaList {
if (!token.hasTrailingTrivia()) {
return Syntax.emptyTriviaList;
}
return triviaScanner.scanTrivia(token, text, /*isTrailing:*/ true);
return triviaScanner.scanTrivia(token, text);
}
function leadingTriviaWidth(token: IScannerToken, text: ISimpleText): number {
@@ -200,15 +187,6 @@ module TypeScript.Scanner {
return lastTokenInfo.leadingTriviaWidth;
}
function trailingTriviaWidth(token: IScannerToken, text: ISimpleText): number {
if (!token.hasTrailingTrivia()) {
return 0;
}
fillSizeInfo(token, text);
return token.fullWidth() - lastTokenInfo.leadingTriviaWidth - lastTokenInfo.width;
}
function tokenIsIncrementallyUnusable(token: IScannerToken): boolean {
// No scanner tokens make their *containing node* incrementally unusable.
// Note: several scanner tokens may themselves be unusable. i.e. if the parser asks
@@ -231,24 +209,21 @@ module TypeScript.Scanner {
}
public childAt(index: number): ISyntaxElement { throw Errors.invalidOperation() }
public accept(visitor: ISyntaxVisitor): any { return visitor.visitToken(this) }
public isIncrementallyUnusable(): boolean { return false; }
public isKeywordConvertedToIdentifier(): boolean { return false; }
public hasSkippedToken(): boolean { return false; }
public fullText(): string { return SyntaxFacts.getText(this.kind); }
public text(): string { return this.fullText(); }
public leadingTrivia(): ISyntaxTriviaList { return Syntax.emptyTriviaList; }
public trailingTrivia(): ISyntaxTriviaList { return Syntax.emptyTriviaList; }
public leadingTriviaWidth(): number { return 0; }
public trailingTriviaWidth(): number { return 0; }
public fullWidth(): number { return fixedWidthTokenLength(this.kind); }
public fullStart(): number { return this._fullStart; }
public hasLeadingTrivia(): boolean { return false; }
public hasTrailingTrivia(): boolean { return false; }
public hasLeadingNewLine(): boolean { return false; }
public hasLeadingSkippedToken(): boolean { return false; }
public hasLeadingComment(): boolean { return false; }
public hasTrailingComment(): boolean { return false; }
public clone(): ISyntaxToken { return new FixedWidthTokenWithNoTrivia(this._fullStart, this.kind); }
}
FixedWidthTokenWithNoTrivia.prototype.childCount = 0;
@@ -271,7 +246,6 @@ module TypeScript.Scanner {
}
public childAt(index: number): ISyntaxElement { throw Errors.invalidOperation() }
public accept(visitor: ISyntaxVisitor): any { return visitor.visitToken(this) }
private syntaxTreeText(text: ISimpleText) {
var result = text || syntaxTree(this).text;
@@ -281,7 +255,6 @@ module TypeScript.Scanner {
public isIncrementallyUnusable(): boolean { return tokenIsIncrementallyUnusable(this); }
public isKeywordConvertedToIdentifier(): boolean { return false; }
public hasSkippedToken(): boolean { return false; }
public fullText(text?: ISimpleText): string {
return fullText(this, this.syntaxTreeText(text));
@@ -293,22 +266,16 @@ module TypeScript.Scanner {
}
public leadingTrivia(text?: ISimpleText): ISyntaxTriviaList { return leadingTrivia(this, this.syntaxTreeText(text)); }
public trailingTrivia(text?: ISimpleText): ISyntaxTriviaList { return trailingTrivia(this, this.syntaxTreeText(text)); }
public leadingTriviaWidth(text?: ISimpleText): number {
return leadingTriviaWidth(this, this.syntaxTreeText(text));
}
public trailingTriviaWidth(text?: ISimpleText): number {
return trailingTriviaWidth(this, this.syntaxTreeText(text));
}
public leadingTriviaWidth(text?: ISimpleText): number { return leadingTriviaWidth(this, this.syntaxTreeText(text)); }
public fullWidth(): number { return largeTokenUnpackFullWidth(this._packedFullWidthAndInfo); }
public fullStart(): number { return this._fullStart; }
public hasLeadingTrivia(): boolean { return largeTokenUnpackHasLeadingTrivia(this._packedFullWidthAndInfo); }
public hasTrailingTrivia(): boolean { return largeTokenUnpackHasTrailingTrivia(this._packedFullWidthAndInfo); }
public hasLeadingNewLine(): boolean { return largeTokenUnpackHasLeadingNewLine(this._packedFullWidthAndInfo); }
public hasLeadingComment(): boolean { return largeTokenUnpackHasLeadingComment(this._packedFullWidthAndInfo); }
public hasTrailingComment(): boolean { return largeTokenUnpackHasTrailingComment(this._packedFullWidthAndInfo); }
public hasLeadingSkippedToken(): boolean { return false; }
public clone(): ISyntaxToken { return new LargeScannerToken(this._fullStart, this.kind, this._packedFullWidthAndInfo, this.cachedText); }
}
LargeScannerToken.prototype.childCount = 0;
@@ -319,12 +286,11 @@ module TypeScript.Scanner {
interface TokenInfo {
leadingTriviaWidth: number;
width: number;
}
interface IScannerInternal extends IScanner {
fillTokenInfo(token: IScannerToken, text: ISimpleText, tokenInfo: TokenInfo): void;
scanTrivia(token: IScannerToken, text: ISimpleText, isTrailing: boolean): ISyntaxTriviaList;
scanTrivia(token: IScannerToken, text: ISimpleText): ISyntaxTriviaList;
}
export interface IScanner {
@@ -367,15 +333,13 @@ module TypeScript.Scanner {
function scan(allowContextualToken: boolean): ISyntaxToken {
var fullStart = index;
var leadingTriviaInfo = scanTriviaInfo(/*isTrailing: */ false);
var leadingTriviaInfo = scanTriviaInfo();
var start = index;
var kindAndIsVariableWidth = scanSyntaxKind(allowContextualToken);
var end = index;
var trailingTriviaInfo = scanTriviaInfo(/*isTrailing: */true);
var fullWidth = index - fullStart;
var fullEnd = index;
var fullWidth = fullEnd - fullStart;
// If we have no trivia, and we are a fixed width token kind, and our size isn't too
// large, and we're a real fixed width token (and not something like "\u0076ar").
@@ -383,28 +347,21 @@ module TypeScript.Scanner {
var isFixedWidth = kind >= SyntaxKind.FirstFixedWidth && kind <= SyntaxKind.LastFixedWidth &&
((kindAndIsVariableWidth & ScannerConstants.IsVariableWidthMask) === 0);
if (isFixedWidth &&
leadingTriviaInfo === 0 && trailingTriviaInfo === 0) {
if (isFixedWidth && leadingTriviaInfo === 0) {
return new FixedWidthTokenWithNoTrivia(fullStart, kind);
}
else {
var packedFullWidthAndInfo = largeTokenPackData(fullWidth, leadingTriviaInfo, trailingTriviaInfo);
var cachedText = isFixedWidth ? undefined : text.substr(start, end - start);
var packedFullWidthAndInfo = largeTokenPackData(fullWidth, leadingTriviaInfo);
var cachedText = isFixedWidth ? undefined : text.substr(start, fullEnd - start);
return new LargeScannerToken(fullStart, kind, packedFullWidthAndInfo, cachedText);
}
}
function scanTrivia(parent: IScannerToken, text: ISimpleText, isTrailing: boolean): ISyntaxTriviaList {
function scanTrivia(parent: IScannerToken, text: ISimpleText): ISyntaxTriviaList {
var tokenFullStart = parent.fullStart();
var tokenStart = tokenFullStart + leadingTriviaWidth(parent, text)
if (isTrailing) {
reset(text, tokenStart + parent.text().length, tokenFullStart + parent.fullWidth());
}
else {
reset(text, tokenFullStart, tokenStart);
}
reset(text, tokenFullStart, tokenStart);
// Debug.assert(length > 0);
// Keep this exactly in sync with scanTriviaInfo
@@ -461,15 +418,7 @@ module TypeScript.Scanner {
case CharacterCodes.paragraphSeparator:
case CharacterCodes.lineSeparator:
trivia.push(scanLineTerminatorSequenceTrivia(ch));
// If we're consuming leading trivia, then we will continue consuming more
// trivia (including newlines) up to the first token we see. If we're
// consuming trailing trivia, then we break after the first newline we see.
if (!isTrailing) {
continue;
}
break;
continue;
default:
throw Errors.invalidOperation();
@@ -486,7 +435,7 @@ module TypeScript.Scanner {
// Returns 0 if there was no trivia, or 1 if there was trivia. Returned as an int instead
// of a boolean because we'll need a numerical value later on to store in our tokens.
function scanTriviaInfo(isTrailing: boolean): number {
function scanTriviaInfo(): number {
// Keep this exactly in sync with scanTrivia
var result = 0;
var _end = end;
@@ -516,14 +465,6 @@ module TypeScript.Scanner {
// we have trivia
result |= ScannerConstants.NewlineTrivia;
// If we're consuming leading trivia, then we will continue consuming more
// trivia (including newlines) up to the first token we see. If we're
// consuming trailing trivia, then we break after the first newline we see.
if (isTrailing) {
return result;
}
continue;
case CharacterCodes.slash:
@@ -1461,14 +1402,10 @@ module TypeScript.Scanner {
var fullEnd = fullStart + token.fullWidth();
reset(text, fullStart, fullEnd);
scanTriviaInfo(/*isTrailing: */ false);
scanTriviaInfo();
var start = index;
scanSyntaxKind(isContextualToken(token));
var end = index;
tokenInfo.leadingTriviaWidth = start - fullStart;
tokenInfo.width = end - start;
}
reset(text, 0, text.length());

98
src/services/syntax/syntax.ts
View File

@@ -9,7 +9,7 @@ module TypeScript.Syntax {
if (isToken(child)) {
var token = <ISyntaxToken>child;
// If a token is skipped, return true. Or if it is a missing token. The only empty token that is not missing is EOF
if (token.hasSkippedToken() || (width(token) === 0 && token.kind !== SyntaxKind.EndOfFileToken)) {
if (token.hasLeadingSkippedToken() || (fullWidth(token) === 0 && token.kind !== SyntaxKind.EndOfFileToken)) {
return true;
}
}
@@ -49,78 +49,6 @@ module TypeScript.Syntax {
return false;
}
export function isEntirelyInsideComment(sourceUnit: SourceUnitSyntax, position: number): boolean {
var positionedToken = findToken(sourceUnit, position);
var fullStart = positionedToken.fullStart();
var triviaList: ISyntaxTriviaList = undefined;
var lastTriviaBeforeToken: ISyntaxTrivia = undefined;
if (positionedToken.kind === SyntaxKind.EndOfFileToken) {
// Check if the trivia is leading on the EndOfFile token
if (positionedToken.hasLeadingTrivia()) {
triviaList = positionedToken.leadingTrivia();
}
// Or trailing on the previous token
else {
positionedToken = previousToken(positionedToken);
if (positionedToken) {
if (positionedToken && positionedToken.hasTrailingTrivia()) {
triviaList = positionedToken.trailingTrivia();
fullStart = end(positionedToken);
}
}
}
}
else {
if (position <= (fullStart + positionedToken.leadingTriviaWidth())) {
triviaList = positionedToken.leadingTrivia();
}
else if (position >= (fullStart + width(positionedToken))) {
triviaList = positionedToken.trailingTrivia();
fullStart = end(positionedToken);
}
}
if (triviaList) {
// Try to find the trivia matching the position
for (var i = 0, n = triviaList.count(); i < n; i++) {
var trivia = triviaList.syntaxTriviaAt(i);
if (position <= fullStart) {
// Moved passed the trivia we need
break;
}
else if (position <= fullStart + trivia.fullWidth() && trivia.isComment()) {
// Found the comment trivia we were looking for
lastTriviaBeforeToken = trivia;
break;
}
fullStart += trivia.fullWidth();
}
}
return lastTriviaBeforeToken && isEntirelyInsideCommentTrivia(lastTriviaBeforeToken, fullStart, position);
}
export function isEntirelyInStringOrRegularExpressionLiteral(sourceUnit: SourceUnitSyntax, position: number): boolean {
var positionedToken = findToken(sourceUnit, position);
if (positionedToken) {
if (positionedToken.kind === SyntaxKind.EndOfFileToken) {
// EndOfFile token, enusre it did not follow an unterminated string literal
positionedToken = previousToken(positionedToken);
return positionedToken && positionedToken.trailingTriviaWidth() === 0 && isUnterminatedStringLiteral(positionedToken);
}
else if (position > start(positionedToken)) {
// Ensure position falls enterily within the literal if it is terminated, or the line if it is not
return (position < end(positionedToken) && (positionedToken.kind === TypeScript.SyntaxKind.StringLiteral || positionedToken.kind === TypeScript.SyntaxKind.RegularExpressionLiteral)) ||
(position <= end(positionedToken) && isUnterminatedStringLiteral(positionedToken));
}
}
return false;
}
export function getAncestorOfKind(positionedToken: ISyntaxElement, kind: SyntaxKind): ISyntaxElement {
while (positionedToken && positionedToken.parent) {
if (positionedToken.parent.kind === kind) {
@@ -203,32 +131,10 @@ module TypeScript.Syntax {
// Debug.assert(position < positionedToken.fullEnd() || positionedToken.token().tokenKind === SyntaxKind.EndOfFileToken);
// if position is after the end of the token, then this token is the token on the left.
if (width(positionedToken) > 0 && position >= end(positionedToken)) {
if (width(positionedToken) > 0 && position >= fullEnd(positionedToken)) {
return positionedToken;
}
return previousToken(positionedToken);
}
export function firstTokenInLineContainingPosition(syntaxTree: SyntaxTree, position: number): ISyntaxToken {
var current = findToken(syntaxTree.sourceUnit(), position);
while (true) {
if (isFirstTokenInLine(current, syntaxTree.lineMap())) {
break;
}
current = previousToken(current);
}
return current;
}
function isFirstTokenInLine(token: ISyntaxToken, lineMap: LineMap): boolean {
var _previousToken = previousToken(token);
if (_previousToken === undefined) {
return true;
}
return lineMap.getLineNumberFromPosition(end(_previousToken)) !== lineMap.getLineNumberFromPosition(start(token));
}
}

56
src/services/syntax/syntaxElement.ts
View File

@@ -70,48 +70,6 @@ module TypeScript {
throw Errors.invalidOperation();
}
export function findSkippedTokenInPositionedToken(positionedToken: ISyntaxToken, position: number): ISyntaxToken {
var positionInLeadingTriviaList = (position < start(positionedToken));
return findSkippedTokenInTriviaList(positionedToken, position, /*lookInLeadingTriviaList*/ positionInLeadingTriviaList);
}
export function findSkippedTokenInLeadingTriviaList(positionedToken: ISyntaxToken, position: number): ISyntaxToken {
return findSkippedTokenInTriviaList(positionedToken, position, /*lookInLeadingTriviaList*/ true);
}
export function findSkippedTokenInTrailingTriviaList(positionedToken: ISyntaxToken, position: number): ISyntaxToken {
return findSkippedTokenInTriviaList(positionedToken, position, /*lookInLeadingTriviaList*/ false);
}
function findSkippedTokenInTriviaList(positionedToken: ISyntaxToken, position: number, lookInLeadingTriviaList: boolean): ISyntaxToken {
var triviaList: TypeScript.ISyntaxTriviaList = undefined;
var fullStart: number;
if (lookInLeadingTriviaList) {
triviaList = positionedToken.leadingTrivia();
fullStart = positionedToken.fullStart();
}
else {
triviaList = positionedToken.trailingTrivia();
fullStart = end(positionedToken);
}
if (triviaList && triviaList.hasSkippedToken()) {
for (var i = 0, n = triviaList.count(); i < n; i++) {
var trivia = triviaList.syntaxTriviaAt(i);
var triviaWidth = trivia.fullWidth();
if (trivia.isSkippedToken() && position >= fullStart && position <= fullStart + triviaWidth) {
return trivia.skippedToken();
}
fullStart += triviaWidth;
}
}
return undefined;
}
function findTokenWorker(element: ISyntaxElement, elementPosition: number, position: number): ISyntaxToken {
if (isList(element)) {
return findTokenInList(<ISyntaxNodeOrToken[]>element, elementPosition, position);
@@ -246,11 +204,6 @@ module TypeScript {
return token ? token.leadingTriviaWidth(text) : 0;
}
export function trailingTriviaWidth(element: ISyntaxElement, text?: ISimpleText): number {
var token = lastToken(element);
return token ? token.trailingTriviaWidth(text) : 0;
}
export function firstToken(element: ISyntaxElement): ISyntaxToken {
if (element) {
var kind = element.kind;
@@ -387,16 +340,11 @@ module TypeScript {
return token ? token.fullStart() + token.leadingTriviaWidth(text) : -1;
}
export function end(element: ISyntaxElement, text?: ISimpleText): number {
var token = isToken(element) ? <ISyntaxToken>element : lastToken(element);
return token ? fullEnd(token) - token.trailingTriviaWidth(text) : -1;
}
export function width(element: ISyntaxElement, text?: ISimpleText): number {
if (isToken(element)) {
return (<ISyntaxToken>element).text().length;
}
return fullWidth(element) - leadingTriviaWidth(element, text) - trailingTriviaWidth(element, text);
return fullWidth(element) - leadingTriviaWidth(element, text);
}
export function fullEnd(element: ISyntaxElement): number {
@@ -413,7 +361,7 @@ module TypeScript {
}
var lineMap = text.lineMap();
return lineMap.getLineNumberFromPosition(end(token1, text)) !== lineMap.getLineNumberFromPosition(start(token2, text));
return lineMap.getLineNumberFromPosition(fullEnd(token1)) !== lineMap.getLineNumberFromPosition(start(token2, text));
}
export interface ISyntaxElement {

87
src/services/syntax/syntaxToken.ts
View File

@@ -16,17 +16,12 @@ module TypeScript {
fullText(text?: ISimpleText): string;
hasLeadingTrivia(): boolean;
hasTrailingTrivia(): boolean;
hasLeadingNewLine(): boolean;
hasLeadingComment(): boolean;
hasTrailingComment(): boolean;
hasSkippedToken(): boolean;
hasLeadingSkippedToken(): boolean;
leadingTrivia(text?: ISimpleText): ISyntaxTriviaList;
trailingTrivia(text?: ISimpleText): ISyntaxTriviaList;
leadingTriviaWidth(text?: ISimpleText): number;
trailingTriviaWidth(text?: ISimpleText): number;
// True if this was a keyword that the parser converted to an identifier. i.e. if you have
// x.public
@@ -284,7 +279,7 @@ module TypeScript {
module TypeScript.Syntax {
export function realizeToken(token: ISyntaxToken, text: ISimpleText): ISyntaxToken {
return new RealizedToken(token.fullStart(), token.kind, token.isKeywordConvertedToIdentifier(), token.leadingTrivia(text), token.text(), token.trailingTrivia(text));
return new RealizedToken(token.fullStart(), token.kind, token.isKeywordConvertedToIdentifier(), token.leadingTrivia(text), token.text());
}
export function convertKeywordToIdentifier(token: ISyntaxToken): ISyntaxToken {
@@ -292,11 +287,7 @@ module TypeScript.Syntax {
}
export function withLeadingTrivia(token: ISyntaxToken, leadingTrivia: ISyntaxTriviaList, text: ISimpleText): ISyntaxToken {
return new RealizedToken(token.fullStart(), token.kind, token.isKeywordConvertedToIdentifier(), leadingTrivia, token.text(), token.trailingTrivia(text));
}
export function withTrailingTrivia(token: ISyntaxToken, trailingTrivia: ISyntaxTriviaList, text: ISimpleText): ISyntaxToken {
return new RealizedToken(token.fullStart(), token.kind, token.isKeywordConvertedToIdentifier(), token.leadingTrivia(text), token.text(), trailingTrivia);
return new RealizedToken(token.fullStart(), token.kind, token.isKeywordConvertedToIdentifier(), leadingTrivia, token.text());
}
export function emptyToken(kind: SyntaxKind): ISyntaxToken {
@@ -317,7 +308,6 @@ module TypeScript.Syntax {
}
public childAt(index: number): ISyntaxElement { throw Errors.invalidOperation() }
public accept(visitor: ISyntaxVisitor): any { return visitor.visitToken(this) }
public clone(): ISyntaxToken {
return new EmptyToken(this.kind);
@@ -405,16 +395,12 @@ module TypeScript.Syntax {
public fullText(): string { return ""; }
public hasLeadingTrivia() { return false; }
public hasTrailingTrivia() { return false; }
public hasLeadingNewLine() { return false; }
public hasLeadingComment() { return false; }
public hasTrailingComment() { return false; }
public hasSkippedToken() { return false; }
public hasLeadingSkippedToken() { return false; }
public leadingTriviaWidth() { return 0; }
public trailingTriviaWidth() { return 0; }
public leadingTrivia(): ISyntaxTriviaList { return Syntax.emptyTriviaList; }
public trailingTrivia(): ISyntaxTriviaList { return Syntax.emptyTriviaList; }
}
EmptyToken.prototype.childCount = 0;
@@ -425,7 +411,6 @@ module TypeScript.Syntax {
private _isKeywordConvertedToIdentifier: boolean;
private _leadingTrivia: ISyntaxTriviaList;
private _text: string;
private _trailingTrivia: ISyntaxTriviaList;
public parent: ISyntaxElement;
public childCount: number;
@@ -434,22 +419,16 @@ module TypeScript.Syntax {
public kind: SyntaxKind,
isKeywordConvertedToIdentifier: boolean,
leadingTrivia: ISyntaxTriviaList,
text: string,
trailingTrivia: ISyntaxTriviaList) {
text: string) {
this._fullStart = fullStart;
this._isKeywordConvertedToIdentifier = isKeywordConvertedToIdentifier;
this._text = text;
this._leadingTrivia = leadingTrivia.clone();
this._trailingTrivia = trailingTrivia.clone();
if (!this._leadingTrivia.isShared()) {
this._leadingTrivia.parent = this;
}
if (!this._trailingTrivia.isShared()) {
this._trailingTrivia.parent = this;
}
}
public setFullStart(fullStart: number): void {
@@ -457,10 +436,9 @@ module TypeScript.Syntax {
}
public childAt(index: number): ISyntaxElement { throw Errors.invalidOperation() }
public accept(visitor: ISyntaxVisitor): any { return visitor.visitToken(this) }
public clone(): ISyntaxToken {
return new RealizedToken(this._fullStart, this.kind, this._isKeywordConvertedToIdentifier, this._leadingTrivia, this._text, this._trailingTrivia);
return new RealizedToken(this._fullStart, this.kind, this._isKeywordConvertedToIdentifier, this._leadingTrivia, this._text);
}
// Realized tokens are created from the parser. They are *never* incrementally reusable.
@@ -471,23 +449,18 @@ module TypeScript.Syntax {
}
public fullStart(): number { return this._fullStart; }
public fullWidth(): number { return this._leadingTrivia.fullWidth() + this._text.length + this._trailingTrivia.fullWidth(); }
public fullWidth(): number { return this._leadingTrivia.fullWidth() + this._text.length; }
public text(): string { return this._text; }
public fullText(): string { return this._leadingTrivia.fullText() + this.text() + this._trailingTrivia.fullText(); }
public fullText(): string { return this._leadingTrivia.fullText() + this.text(); }
public hasLeadingTrivia(): boolean { return this._leadingTrivia.count() > 0; }
public hasTrailingTrivia(): boolean { return this._trailingTrivia.count() > 0; }
public hasLeadingNewLine(): boolean { return this._leadingTrivia.hasNewLine(); }
public hasLeadingComment(): boolean { return this._leadingTrivia.hasComment(); }
public hasTrailingComment(): boolean { return this._trailingTrivia.hasComment(); }
public leadingTriviaWidth(): number { return this._leadingTrivia.fullWidth(); }
public trailingTriviaWidth(): number { return this._trailingTrivia.fullWidth(); }
public hasSkippedToken(): boolean { return this._leadingTrivia.hasSkippedToken() || this._trailingTrivia.hasSkippedToken(); }
public hasLeadingSkippedToken(): boolean { return this._leadingTrivia.hasSkippedToken(); }
public leadingTrivia(): ISyntaxTriviaList { return this._leadingTrivia; }
public trailingTrivia(): ISyntaxTriviaList { return this._trailingTrivia; }
public leadingTriviaWidth(): number { return this._leadingTrivia.fullWidth(); }
}
RealizedToken.prototype.childCount = 0;
@@ -506,7 +479,6 @@ module TypeScript.Syntax {
}
public childAt(index: number): ISyntaxElement { throw Errors.invalidOperation() }
public accept(visitor: ISyntaxVisitor): any { return visitor.visitToken(this) }
public fullStart(): number {
return this.underlyingToken.fullStart();
@@ -530,25 +502,10 @@ module TypeScript.Syntax {
return this.underlyingToken.fullText(this.syntaxTreeText(text));
}
public hasLeadingTrivia(): boolean {
return this.underlyingToken.hasLeadingTrivia();
}
public hasTrailingTrivia(): boolean {
return this.underlyingToken.hasTrailingTrivia();
}
public hasLeadingComment(): boolean {
return this.underlyingToken.hasLeadingComment();
}
public hasTrailingComment(): boolean {
return this.underlyingToken.hasTrailingComment();
}
public hasSkippedToken(): boolean {
return this.underlyingToken.hasSkippedToken();
}
public hasLeadingTrivia(): boolean { return this.underlyingToken.hasLeadingTrivia(); }
public hasLeadingNewLine(): boolean { return this.underlyingToken.hasLeadingNewLine(); }
public hasLeadingComment(): boolean { return this.underlyingToken.hasLeadingComment(); }
public hasLeadingSkippedToken(): boolean { return this.underlyingToken.hasLeadingSkippedToken(); }
public leadingTrivia(text?: ISimpleText): ISyntaxTriviaList {
var result = this.underlyingToken.leadingTrivia(this.syntaxTreeText(text));
@@ -556,20 +513,10 @@ module TypeScript.Syntax {
return result;
}
public trailingTrivia(text?: ISimpleText): ISyntaxTriviaList {
var result = this.underlyingToken.trailingTrivia(this.syntaxTreeText(text));
result.parent = this;
return result;
}
public leadingTriviaWidth(text?: ISimpleText): number {
return this.underlyingToken.leadingTriviaWidth(this.syntaxTreeText(text));
}
public trailingTriviaWidth(text?: ISimpleText): number {
return this.underlyingToken.trailingTriviaWidth(this.syntaxTreeText(text));
}
public isKeywordConvertedToIdentifier(): boolean {
return true;
}

1
src/services/syntax/syntaxTrivia.ts
View File

@@ -127,7 +127,6 @@ module TypeScript.Syntax {
export function skippedTokenTrivia(token: ISyntaxToken, text: ISimpleText): ISyntaxTrivia {
Debug.assert(!token.hasLeadingTrivia());
Debug.assert(!token.hasTrailingTrivia());
Debug.assert(token.fullWidth() > 0);
return new SkippedTokenTrivia(token, token.fullText(text));
}

2
src/services/syntax/syntaxUtilities.ts
View File

@@ -36,7 +36,7 @@ module TypeScript {
}
var lineMap = text.lineMap();
var tokenLine = lineMap.getLineNumberFromPosition(end(token, text));
var tokenLine = lineMap.getLineNumberFromPosition(fullEnd(token));
var nextTokenLine = lineMap.getLineNumberFromPosition(start(_nextToken, text));
return tokenLine !== nextTokenLine;

8
src/services/syntax/testUtilities.ts
View File

@@ -69,10 +69,8 @@ module TypeScript {
token1.fullStart() === token2.fullStart() &&
TypeScript.fullEnd(token1) === TypeScript.fullEnd(token2) &&
TypeScript.start(token1, text1) === TypeScript.start(token2, text2) &&
TypeScript.end(token1, text1) === TypeScript.end(token2, text2) &&
token1.text() === token2.text() &&
triviaListStructuralEquals(token1.leadingTrivia(text1), token2.leadingTrivia(text2)) &&
triviaListStructuralEquals(token1.trailingTrivia(text1), token2.trailingTrivia(text2));
triviaListStructuralEquals(token1.leadingTrivia(text1), token2.leadingTrivia(text2));
}
export function triviaListStructuralEquals(triviaList1: TypeScript.ISyntaxTriviaList, triviaList2: TypeScript.ISyntaxTriviaList): boolean {
@@ -150,10 +148,6 @@ module TypeScript {
return false;
}
if (TypeScript.end(element1) !== TypeScript.end(element2)) {
return false;
}
if (TypeScript.fullEnd(element1) !== TypeScript.fullEnd(element2)) {
return false;
}