From 9cf588d4beb839c41ef1d219b988f2dd2f3adbba Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 5 Dec 2014 08:28:07 -0800 Subject: [PATCH 1/3] Move syntax cursor into its own file. --- src/services/syntax/incrementalParser.ts | 279 ----------------------- src/services/syntax/references.ts | 6 +- src/services/syntax/syntaxCursor.ts | 255 +++++++++++++++++++++ src/services/syntax/syntaxElement.ts | 1 - src/services/syntax/syntaxUtilities.ts | 76 +++--- 5 files changed, 289 insertions(+), 328 deletions(-) create mode 100644 src/services/syntax/syntaxCursor.ts diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index fabea00e9db..5556dfc10da 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -386,270 +386,6 @@ module TypeScript.IncrementalParser { }; } - interface SyntaxCursorPiece { - element: ISyntaxElement; - indexInParent: number - } - - function createSyntaxCursorPiece(element: ISyntaxElement, indexInParent: number) { - return { element: element, indexInParent: indexInParent }; - } - - // Pool syntax cursors so we don't churn too much memory when we need temporary cursors. - // i.e. when we're speculatively parsing, we can cheaply get a pooled cursor and then - // return it when we no longer need it. - var syntaxCursorPool: SyntaxCursor[] = []; - var syntaxCursorPoolCount: number = 0; - - function returnSyntaxCursor(cursor: SyntaxCursor): void { - // Make sure the cursor isn't holding onto any syntax elements. We don't want to leak - // them when we return the cursor to the pool. - cursor.clean(); - - syntaxCursorPool[syntaxCursorPoolCount] = cursor; - syntaxCursorPoolCount++; - } - - function getSyntaxCursor(): SyntaxCursor { - // Get an existing cursor from the pool if we have one. Or create a new one if we don't. - var cursor = syntaxCursorPoolCount > 0 - ? syntaxCursorPool[syntaxCursorPoolCount - 1] - : createSyntaxCursor(); - - if (syntaxCursorPoolCount > 0) { - // If we reused an existing cursor, take it out of the pool so no one else uses it. - syntaxCursorPoolCount--; - syntaxCursorPool[syntaxCursorPoolCount] = undefined; - } - - return cursor; - } - - function cloneSyntaxCursor(cursor: SyntaxCursor): SyntaxCursor { - var newCursor = getSyntaxCursor(); - - // Make the new cursor a *deep* copy of the cursor passed in. This ensures each cursor can - // be moved without affecting the other. - newCursor.deepCopyFrom(cursor); - - return newCursor; - } - - interface SyntaxCursor { - pieces: SyntaxCursorPiece[]; - - clean(): void; - isFinished(): boolean; - moveToFirstChild(): void; - moveToFirstToken(): void; - moveToNextSibling(): void; - currentNodeOrToken(): ISyntaxNodeOrToken; - currentNode(): ISyntaxNode; - currentToken(): ISyntaxToken; - pushElement(element: ISyntaxElement, indexInParent: number): void; - deepCopyFrom(other: SyntaxCursor): void; - } - - function createSyntaxCursor(): SyntaxCursor { - // Our list of path pieces. The piece pointed to by 'currentPieceIndex' must be a node or - // token. However, pieces earlier than that may point to list nodes. - // - // For perf we reuse pieces as much as possible. i.e. instead of popping items off the - // list, we just will change currentPieceIndex so we can reuse that piece later. - var pieces: SyntaxCursorPiece[] = []; - var currentPieceIndex: number = -1; - - // Cleans up this cursor so that it doesn't have any references to actual syntax nodes. - // This sould be done before returning the cursor to the pool so that the Parser module - // doesn't unnecessarily keep old syntax trees alive. - function clean(): void { - for (var i = 0, n = pieces.length; i < n; i++) { - var piece = pieces[i]; - - if (piece.element === undefined) { - break; - } - - piece.element = undefined; - piece.indexInParent = -1; - } - - currentPieceIndex = -1; - } - - // Makes this cursor into a deep copy of the cursor passed in. - function deepCopyFrom(other: SyntaxCursor): void { - for (var i = 0, n = other.pieces.length; i < n; i++) { - var piece = other.pieces[i]; - - if (piece.element === undefined) { - break; - } - - pushElement(piece.element, piece.indexInParent); - } - } - - function isFinished(): boolean { - return currentPieceIndex < 0; - } - - function currentNodeOrToken(): ISyntaxNodeOrToken { - if (isFinished()) { - return undefined; - } - - var result = pieces[currentPieceIndex].element; - - // The current element must always be a node or a token. - return result; - } - - function currentNode(): ISyntaxNode { - var element = currentNodeOrToken(); - return isNode(element) ? element : undefined; - } - - function isEmptyList(element: ISyntaxElement) { - return isList(element) && (element).length === 0; - } - - function moveToFirstChild() { - var nodeOrToken = currentNodeOrToken(); - if (nodeOrToken === undefined) { - return; - } - - if (isToken(nodeOrToken)) { - // If we're already on a token, there's nothing to do. - return; - } - - // Either the node has some existent child, then move to it. if it doesn't, then it's - // an empty node. Conceptually the first child of an empty node is really just the - // next sibling of the empty node. - for (var i = 0, n = childCount(nodeOrToken); i < n; i++) { - var child = childAt(nodeOrToken, i); - if (child && !isEmptyList(child)) { - // Great, we found a real child. Push that. - pushElement(child, /*indexInParent:*/ i); - - // If it was a list, make sure we're pointing at its first element. We know we - // must have one because this is a non-shared list. - moveToFirstChildIfList(); - return; - } - } - - // This element must have been an empty node. Moving to its 'first child' is equivalent to just - // moving to the next sibling. - moveToNextSibling(); - } - - function moveToNextSibling(): void { - while (!isFinished()) { - // first look to our parent and see if it has a sibling of us that we can move to. - var currentPiece = pieces[currentPieceIndex]; - var parent = currentPiece.element.parent; - - // We start searching at the index one past our own index in the parent. - for (var i = currentPiece.indexInParent + 1, n = childCount(parent); i < n; i++) { - var sibling = childAt(parent, i); - - if (sibling && !isEmptyList(sibling)) { - // We found a good sibling that we can move to. Just reuse our existing piece - // so we don't have to push/pop. - currentPiece.element = sibling; - currentPiece.indexInParent = i; - - // The sibling might have been a list. Move to it's first child. - moveToFirstChildIfList(); - return; - } - } - - // Didn't have a sibling for this element. Go up to our parent and get its sibling. - - // Clear the data from the old piece. We don't want to keep any elements around - // unintentionally. - currentPiece.element = undefined; - currentPiece.indexInParent = -1; - - // Point at the parent. if we move past the top of the path, then we're finished. - currentPieceIndex--; - } - } - - function moveToFirstChildIfList(): void { - var element = pieces[currentPieceIndex].element; - - if (isList(element)) { - // We cannot ever get an empty list in our piece path. Empty lists are 'shared' and - // we make sure to filter that out before pushing any children. - pushElement(childAt(element, 0), /*indexInParent:*/ 0); - } - } - - function pushElement(element: ISyntaxElement, indexInParent: number): void { - currentPieceIndex++; - - // Reuse an existing piece if we have one. Otherwise, push a new piece to our list. - if (currentPieceIndex === pieces.length) { - pieces.push(createSyntaxCursorPiece(element, indexInParent)); - } - else { - var piece = pieces[currentPieceIndex]; - piece.element = element; - piece.indexInParent = indexInParent; - } - } - - function moveToFirstToken(): void { - while (!isFinished()) { - var element = pieces[currentPieceIndex].element; - if (isNode(element)) { - moveToFirstChild(); - continue; - } - - return; - } - } - - function currentToken(): ISyntaxToken { - moveToFirstToken(); - - var element = currentNodeOrToken(); - return element; - } - - return { - pieces: pieces, - clean: clean, - isFinished: isFinished, - moveToFirstChild: moveToFirstChild, - moveToFirstToken: moveToFirstToken, - moveToNextSibling: moveToNextSibling, - currentNodeOrToken: currentNodeOrToken, - currentNode: currentNode, - currentToken: currentToken, - pushElement: pushElement, - deepCopyFrom: deepCopyFrom - }; - } - - // A simple walker we use to hit all the tokens of a node and update their positions when they - // are reused in a different location because of an incremental parse. - - class TokenCollectorWalker extends SyntaxWalker { - public tokens: ISyntaxToken[] = []; - - public visitToken(token: ISyntaxToken): void { - this.tokens.push(token); - } - } - - var tokenCollectorWalker = new TokenCollectorWalker(); function updateTokenPositionsAndMarkElements(element: ISyntaxElement, changeStart: number, changeRangeOldEnd: number, delta: number, fullStart: number): void { // First, try to skip past any elements that we dont' need to move. We don't need to // move any elements that don't start after the end of the change range. @@ -724,21 +460,6 @@ module TypeScript.IncrementalParser { } } - function getTokens(node: ISyntaxNode): ISyntaxToken[] { - var tokens = node.__cachedTokens; - if (!tokens) { - tokens = []; - tokenCollectorWalker.tokens = tokens; - - visitNodeOrToken(tokenCollectorWalker, node); - - node.__cachedTokens = tokens; - tokenCollectorWalker.tokens = undefined; - } - - return tokens; - } - export function parse(oldSyntaxTree: SyntaxTree, textChangeRange: TextChangeRange, newText: ISimpleText): SyntaxTree { if (textChangeRange.isUnchanged()) { return oldSyntaxTree; diff --git a/src/services/syntax/references.ts b/src/services/syntax/references.ts index e425fda3936..38b5fd9a160 100644 --- a/src/services/syntax/references.ts +++ b/src/services/syntax/references.ts @@ -27,13 +27,15 @@ /// /// /// -/// /// /// // SyntaxInformationMap depends on SyntaxWalker // /// +// SyntaxUtilities depends on SyntaxWalker +/// + /// // Concrete nodes depend on the parser. @@ -44,3 +46,5 @@ /// /// +/// +/// diff --git a/src/services/syntax/syntaxCursor.ts b/src/services/syntax/syntaxCursor.ts new file mode 100644 index 00000000000..3c89782144d --- /dev/null +++ b/src/services/syntax/syntaxCursor.ts @@ -0,0 +1,255 @@ +/// + +module TypeScript.IncrementalParser { + interface SyntaxCursorPiece { + element: ISyntaxElement; + indexInParent: number + } + + function createSyntaxCursorPiece(element: ISyntaxElement, indexInParent: number) { + return { element: element, indexInParent: indexInParent }; + } + + // Pool syntax cursors so we don't churn too much memory when we need temporary cursors. + // i.e. when we're speculatively parsing, we can cheaply get a pooled cursor and then + // return it when we no longer need it. + var syntaxCursorPool: SyntaxCursor[] = []; + var syntaxCursorPoolCount: number = 0; + + export function returnSyntaxCursor(cursor: SyntaxCursor): void { + // Make sure the cursor isn't holding onto any syntax elements. We don't want to leak + // them when we return the cursor to the pool. + cursor.clean(); + + syntaxCursorPool[syntaxCursorPoolCount] = cursor; + syntaxCursorPoolCount++; + } + + export function getSyntaxCursor(): SyntaxCursor { + // Get an existing cursor from the pool if we have one. Or create a new one if we don't. + var cursor = syntaxCursorPoolCount > 0 + ? syntaxCursorPool[syntaxCursorPoolCount - 1] + : createSyntaxCursor(); + + if (syntaxCursorPoolCount > 0) { + // If we reused an existing cursor, take it out of the pool so no one else uses it. + syntaxCursorPoolCount--; + syntaxCursorPool[syntaxCursorPoolCount] = undefined; + } + + return cursor; + } + + export function cloneSyntaxCursor(cursor: SyntaxCursor): SyntaxCursor { + var newCursor = getSyntaxCursor(); + + // Make the new cursor a *deep* copy of the cursor passed in. This ensures each cursor can + // be moved without affecting the other. + newCursor.deepCopyFrom(cursor); + + return newCursor; + } + + interface SyntaxCursor { + pieces: SyntaxCursorPiece[]; + + clean(): void; + isFinished(): boolean; + moveToFirstChild(): void; + moveToFirstToken(): void; + moveToNextSibling(): void; + currentNodeOrToken(): ISyntaxNodeOrToken; + currentNode(): ISyntaxNode; + currentToken(): ISyntaxToken; + pushElement(element: ISyntaxElement, indexInParent: number): void; + deepCopyFrom(other: SyntaxCursor): void; + } + + function createSyntaxCursor(): SyntaxCursor { + // Our list of path pieces. The piece pointed to by 'currentPieceIndex' must be a node or + // token. However, pieces earlier than that may point to list nodes. + // + // For perf we reuse pieces as much as possible. i.e. instead of popping items off the + // list, we just will change currentPieceIndex so we can reuse that piece later. + var pieces: SyntaxCursorPiece[] = []; + var currentPieceIndex: number = -1; + + // Cleans up this cursor so that it doesn't have any references to actual syntax nodes. + // This sould be done before returning the cursor to the pool so that the Parser module + // doesn't unnecessarily keep old syntax trees alive. + function clean(): void { + for (var i = 0, n = pieces.length; i < n; i++) { + var piece = pieces[i]; + + if (piece.element === undefined) { + break; + } + + piece.element = undefined; + piece.indexInParent = -1; + } + + currentPieceIndex = -1; + } + + // Makes this cursor into a deep copy of the cursor passed in. + function deepCopyFrom(other: SyntaxCursor): void { + for (var i = 0, n = other.pieces.length; i < n; i++) { + var piece = other.pieces[i]; + + if (piece.element === undefined) { + break; + } + + pushElement(piece.element, piece.indexInParent); + } + } + + function isFinished(): boolean { + return currentPieceIndex < 0; + } + + function currentNodeOrToken(): ISyntaxNodeOrToken { + if (isFinished()) { + return undefined; + } + + var result = pieces[currentPieceIndex].element; + + // The current element must always be a node or a token. + return result; + } + + function currentNode(): ISyntaxNode { + var element = currentNodeOrToken(); + return isNode(element) ? element : undefined; + } + + function isEmptyList(element: ISyntaxElement) { + return isList(element) && (element).length === 0; + } + + function moveToFirstChild() { + var nodeOrToken = currentNodeOrToken(); + if (nodeOrToken === undefined) { + return; + } + + if (isToken(nodeOrToken)) { + // If we're already on a token, there's nothing to do. + return; + } + + // Either the node has some existent child, then move to it. if it doesn't, then it's + // an empty node. Conceptually the first child of an empty node is really just the + // next sibling of the empty node. + for (var i = 0, n = childCount(nodeOrToken); i < n; i++) { + var child = childAt(nodeOrToken, i); + if (child && !isEmptyList(child)) { + // Great, we found a real child. Push that. + pushElement(child, /*indexInParent:*/ i); + + // If it was a list, make sure we're pointing at its first element. We know we + // must have one because this is a non-shared list. + moveToFirstChildIfList(); + return; + } + } + + // This element must have been an empty node. Moving to its 'first child' is equivalent to just + // moving to the next sibling. + moveToNextSibling(); + } + + function moveToNextSibling(): void { + while (!isFinished()) { + // first look to our parent and see if it has a sibling of us that we can move to. + var currentPiece = pieces[currentPieceIndex]; + var parent = currentPiece.element.parent; + + // We start searching at the index one past our own index in the parent. + for (var i = currentPiece.indexInParent + 1, n = childCount(parent); i < n; i++) { + var sibling = childAt(parent, i); + + if (sibling && !isEmptyList(sibling)) { + // We found a good sibling that we can move to. Just reuse our existing piece + // so we don't have to push/pop. + currentPiece.element = sibling; + currentPiece.indexInParent = i; + + // The sibling might have been a list. Move to it's first child. + moveToFirstChildIfList(); + return; + } + } + + // Didn't have a sibling for this element. Go up to our parent and get its sibling. + + // Clear the data from the old piece. We don't want to keep any elements around + // unintentionally. + currentPiece.element = undefined; + currentPiece.indexInParent = -1; + + // Point at the parent. if we move past the top of the path, then we're finished. + currentPieceIndex--; + } + } + + function moveToFirstChildIfList(): void { + var element = pieces[currentPieceIndex].element; + + if (isList(element)) { + // We cannot ever get an empty list in our piece path. Empty lists are 'shared' and + // we make sure to filter that out before pushing any children. + pushElement(childAt(element, 0), /*indexInParent:*/ 0); + } + } + + function pushElement(element: ISyntaxElement, indexInParent: number): void { + currentPieceIndex++; + + // Reuse an existing piece if we have one. Otherwise, push a new piece to our list. + if (currentPieceIndex === pieces.length) { + pieces.push(createSyntaxCursorPiece(element, indexInParent)); + } + else { + var piece = pieces[currentPieceIndex]; + piece.element = element; + piece.indexInParent = indexInParent; + } + } + + function moveToFirstToken(): void { + while (!isFinished()) { + var element = pieces[currentPieceIndex].element; + if (isNode(element)) { + moveToFirstChild(); + continue; + } + + return; + } + } + + function currentToken(): ISyntaxToken { + moveToFirstToken(); + + var element = currentNodeOrToken(); + return element; + } + + return { + pieces: pieces, + clean: clean, + isFinished: isFinished, + moveToFirstChild: moveToFirstChild, + moveToFirstToken: moveToFirstToken, + moveToNextSibling: moveToNextSibling, + currentNodeOrToken: currentNodeOrToken, + currentNode: currentNode, + currentToken: currentToken, + pushElement: pushElement, + deepCopyFrom: deepCopyFrom + }; + } +} \ No newline at end of file diff --git a/src/services/syntax/syntaxElement.ts b/src/services/syntax/syntaxElement.ts index a76cf00721b..946ac5dc480 100644 --- a/src/services/syntax/syntaxElement.ts +++ b/src/services/syntax/syntaxElement.ts @@ -378,7 +378,6 @@ module TypeScript { export interface ISyntaxNode extends ISyntaxNodeOrToken { __data: number; - __cachedTokens: ISyntaxToken[]; } export interface IModuleReferenceSyntax extends ISyntaxNode { diff --git a/src/services/syntax/syntaxUtilities.ts b/src/services/syntax/syntaxUtilities.ts index 634296740d4..4b08d3a14b6 100644 --- a/src/services/syntax/syntaxUtilities.ts +++ b/src/services/syntax/syntaxUtilities.ts @@ -11,6 +11,35 @@ module TypeScript { return (element).childAt(index); } + interface ISyntaxNodeInternal extends ISyntaxNode { + __cachedTokens: ISyntaxToken[]; + } + + class TokenCollectorWalker extends SyntaxWalker { + public tokens: ISyntaxToken[] = []; + + public visitToken(token: ISyntaxToken): void { + this.tokens.push(token); + } + } + + var tokenCollectorWalker = new TokenCollectorWalker(); + + export function getTokens(node: ISyntaxNode): ISyntaxToken[] { + var tokens = (node).__cachedTokens; + if (!tokens) { + tokens = []; + tokenCollectorWalker.tokens = tokens; + + visitNodeOrToken(tokenCollectorWalker, node); + + (node).__cachedTokens = tokens; + tokenCollectorWalker.tokens = undefined; + } + + return tokens; + } + export module SyntaxUtilities { export function isAnyFunctionExpressionOrDeclaration(ast: ISyntaxElement): boolean { switch (ast.kind) { @@ -28,19 +57,6 @@ module TypeScript { return false; } - export function isLastTokenOnLine(token: ISyntaxToken, text: ISimpleText): boolean { - var _nextToken = nextToken(token, text); - if (_nextToken === undefined) { - return true; - } - - var lineMap = text.lineMap(); - var tokenLine = lineMap.getLineNumberFromPosition(fullEnd(token)); - var nextTokenLine = lineMap.getLineNumberFromPosition(start(_nextToken, text)); - - return tokenLine !== nextTokenLine; - } - export function isLeftHandSizeExpression(element: ISyntaxElement) { if (element) { switch (element.kind) { @@ -178,21 +194,6 @@ module TypeScript { return false; } - export function isAngleBracket(positionedElement: ISyntaxElement): boolean { - var element = positionedElement; - var parent = positionedElement.parent; - if (parent && (element.kind === SyntaxKind.LessThanToken || element.kind === SyntaxKind.GreaterThanToken)) { - switch (parent.kind) { - case SyntaxKind.TypeArgumentList: - case SyntaxKind.TypeParameterList: - case SyntaxKind.TypeAssertionExpression: - return true; - } - } - - return false; - } - export function getToken(list: ISyntaxToken[], kind: SyntaxKind): ISyntaxToken { for (var i = 0, n = list.length; i < n; i++) { var token = list[i]; @@ -207,24 +208,5 @@ module TypeScript { export function containsToken(list: ISyntaxToken[], kind: SyntaxKind): boolean { return !!SyntaxUtilities.getToken(list, kind); } - - export function hasExportKeyword(moduleElement: IModuleElementSyntax): boolean { - return !!SyntaxUtilities.getExportKeyword(moduleElement); - } - - export function getExportKeyword(moduleElement: IModuleElementSyntax): ISyntaxToken { - switch (moduleElement.kind) { - case SyntaxKind.ModuleDeclaration: - case SyntaxKind.ClassDeclaration: - case SyntaxKind.FunctionDeclaration: - case SyntaxKind.VariableStatement: - case SyntaxKind.EnumDeclaration: - case SyntaxKind.InterfaceDeclaration: - case SyntaxKind.ImportDeclaration: - return SyntaxUtilities.getToken((moduleElement).modifiers, SyntaxKind.ExportKeyword); - default: - return undefined; - } - } } } \ No newline at end of file From 12e90a09efd231654692697d2fb6b555296387d0 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 5 Dec 2014 08:33:10 -0800 Subject: [PATCH 2/3] Simplify parser API. --- src/services/syntax/incrementalParser.ts | 8 -------- src/services/syntax/parser.ts | 20 ++++---------------- src/services/syntax/scanner.ts | 12 ------------ 3 files changed, 4 insertions(+), 36 deletions(-) diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index 5556dfc10da..4ecd75b0001 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -72,13 +72,6 @@ module TypeScript.IncrementalParser { updateTokenPositionsAndMarkElements(oldSourceUnit, _changeRange.span().start(), _changeRange.span().end(), delta, /*fullStart:*/ 0); - function release() { - _scannerParserSource.release(); - _scannerParserSource = undefined; - _oldSourceUnitCursor = undefined; - _isSpeculativelyParsing = false; - } - function extendToAffectedRange(changeRange: TextChangeRange, sourceUnit: SourceUnitSyntax): TextChangeRange { // Consider the following code: // void foo() { /; } @@ -382,7 +375,6 @@ module TypeScript.IncrementalParser { consumeNodeOrToken: consumeNodeOrToken, tryParse: tryParse, tokenDiagnostics: tokenDiagnostics, - release: release }; } diff --git a/src/services/syntax/parser.ts b/src/services/syntax/parser.ts index 74ffa81f39c..ecb3c9a5233 100644 --- a/src/services/syntax/parser.ts +++ b/src/services/syntax/parser.ts @@ -77,26 +77,19 @@ module TypeScript.Parser { // Retrieves the diagnostics generated while the source was producing nodes or tokens. // Should generally only be called after the document has been completely parsed. tokenDiagnostics(): Diagnostic[]; - - release(): void; } // Contains the actual logic to parse typescript/javascript. This is the code that generally // represents the logic necessary to handle all the language grammar constructs. When the // language changes, this should generally only be the place necessary to fix up. function createParseSyntaxTree(): (source: IParserSource, isDeclaration: boolean) => SyntaxTree { - // Name of the file we're parsing. - var fileName: string; - // Underlying source where we pull nodes and tokens from. var source: IParserSource; - var languageVersion: ts.ScriptTarget; - // TODO: do we need to store/restore this when speculative parsing? I don't think so. The // parsing logic already handles storing/restoring this and should work properly even if we're // speculative parsing. - var listParsingState: number = 0; + var listParsingState: ListParsingState = 0; // Whether or not we are in strict parsing mode. All that changes in strict parsing mode is // that some tokens that would be considered identifiers may be considered keywords. When @@ -280,9 +273,7 @@ module TypeScript.Parser { function parseSyntaxTree(_source: IParserSource, isDeclaration: boolean): SyntaxTree { // First, set up our state. - fileName = _source.fileName; source = _source; - languageVersion = source.languageVersion; // Now actually parse the tree. var result = parseSyntaxTreeWorker(isDeclaration); @@ -290,10 +281,7 @@ module TypeScript.Parser { // Now, clear out our state so that our singleton parser doesn't keep things alive. diagnostics = []; contextFlags = 0; - fileName = undefined; - source.release(); source = undefined; - _source = undefined; return result; } @@ -304,7 +292,7 @@ module TypeScript.Parser { var allDiagnostics = source.tokenDiagnostics().concat(diagnostics); allDiagnostics.sort((a: Diagnostic, b: Diagnostic) => a.start() - b.start()); - return new SyntaxTree(sourceUnit, isDeclaration, allDiagnostics, fileName, source.text, languageVersion); + return new SyntaxTree(sourceUnit, isDeclaration, allDiagnostics, source.fileName, source.text, source.languageVersion); } function tryParse(callback: () => T): T { @@ -647,7 +635,7 @@ module TypeScript.Parser { } } - return new Diagnostic(fileName, source.text.lineMap(), start(token, source.text), width(token), diagnosticCode, args); + return new Diagnostic(source.fileName, source.text.lineMap(), start(token, source.text), width(token), diagnosticCode, args); } function getBinaryExpressionPrecedence(tokenKind: SyntaxKind): BinaryExpressionPrecedence { @@ -4397,7 +4385,7 @@ module TypeScript.Parser { function reportUnexpectedTokenDiagnostic(listType: ListParsingState): void { var token = currentToken(); - var diagnostic = new Diagnostic(fileName, source.text.lineMap(), + var diagnostic = new Diagnostic(source.fileName, source.text.lineMap(), start(token, source.text), width(token), DiagnosticCode.Unexpected_token_0_expected, [getExpectedListElementType(listType)]); addDiagnostic(diagnostic); } diff --git a/src/services/syntax/scanner.ts b/src/services/syntax/scanner.ts index c20f1dca779..ff19831e891 100644 --- a/src/services/syntax/scanner.ts +++ b/src/services/syntax/scanner.ts @@ -1456,14 +1456,6 @@ module TypeScript.Scanner { // The scanner we're pulling tokens from. var scanner = createScanner(languageVersion, text, reportDiagnostic); - function release() { - slidingWindow = undefined; - scanner = undefined; - _tokenDiagnostics = []; - lastDiagnostic = undefined; - reportDiagnostic = undefined; - } - function currentNode(): ISyntaxNode { // The normal parser source never returns nodes. They're only returned by the // incremental parser source. @@ -1605,12 +1597,8 @@ module TypeScript.Scanner { currentContextualToken: currentContextualToken, peekToken: peekToken, consumeNodeOrToken: consumeNodeOrToken, - //getRewindPoint: getRewindPoint, - //rewind: rewind, - //releaseRewindPoint: releaseRewindPoint, tryParse: tryParse, tokenDiagnostics: tokenDiagnostics, - release: release, absolutePosition: absolutePosition, }; } From 99827952004dc226551cd7778d06e18d0f344043 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 5 Dec 2014 09:03:39 -0800 Subject: [PATCH 3/3] Rename _skippedTokens to skippedTokens. --- src/services/syntax/parser.ts | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/services/syntax/parser.ts b/src/services/syntax/parser.ts index ecb3c9a5233..be2ab436456 100644 --- a/src/services/syntax/parser.ts +++ b/src/services/syntax/parser.ts @@ -148,7 +148,11 @@ module TypeScript.Parser { // started at. var diagnostics: Diagnostic[] = []; - var _skippedTokens: ISyntaxToken[] = undefined; + // Any tokens we decided to skip during list (when we can't find any parse function that + // will accept the token). These tokens will be attached to the next token we actually + // consume. If there are any skipped tokens at the end of the file, they will be attached + // to the EndOfFile token. + var skippedTokens: ISyntaxToken[] = undefined; function setContextFlag(val: boolean, flag: ParserContextFlags) { if (val) { @@ -299,13 +303,13 @@ module TypeScript.Parser { // See the comments in IParserRewindPoint for the explanation on why we need to store // this data, and what it is used for. var savedDiagnosticsCount = diagnostics.length; - var savedSkippedTokens = _skippedTokens ? _skippedTokens.slice(0) : undefined; + var savedSkippedTokens = skippedTokens ? skippedTokens.slice(0) : undefined; var result = source.tryParse(callback); if (!result) { diagnostics.length = savedDiagnosticsCount; - _skippedTokens = savedSkippedTokens; + skippedTokens = savedSkippedTokens; } return result; @@ -316,7 +320,7 @@ module TypeScript.Parser { // TODO(cyrusn): This may be too conservative. Perhaps we could reuse hte node and // attach the skipped tokens in front? For now though, being conservative is nice and // safe, and likely won't ever affect perf. - if (!_skippedTokens) { + if (!skippedTokens) { var node = source.currentNode(); // We can only reuse a node if it was parsed under the same strict mode that we're @@ -353,8 +357,8 @@ module TypeScript.Parser { } function skipToken(token: ISyntaxToken): void { - _skippedTokens = _skippedTokens || []; - _skippedTokens.push(token); + skippedTokens = skippedTokens || []; + skippedTokens.push(token); // directly tell the source to just consume the token we're skipping. i.e. do not // call 'consumeToken'. Doing so would attempt to add any previous skipped tokens @@ -371,9 +375,9 @@ module TypeScript.Parser { // Now, if we had any skipped tokens, we want to add them to the start of this token // we're consuming. - if (_skippedTokens) { - token = addSkippedTokensBeforeToken(token, _skippedTokens); - _skippedTokens = undefined; + if (skippedTokens) { + token = addSkippedTokensBeforeToken(token, skippedTokens); + skippedTokens = undefined; } return token; @@ -424,7 +428,7 @@ module TypeScript.Parser { } function consumeNode(node: ISyntaxNode): void { - Debug.assert(_skippedTokens === undefined); + Debug.assert(skippedTokens === undefined); source.consumeNodeOrToken(node); } @@ -597,7 +601,7 @@ module TypeScript.Parser { // So, if we have any skipped tokens, then the position of the empty token should be // the position of the first skipped token we have. Otherwise it's just at the position // of the parser. - var fullStart = _skippedTokens ? _skippedTokens[0].fullStart() : source.absolutePosition(); + var fullStart = skippedTokens ? skippedTokens[0].fullStart() : source.absolutePosition(); return Syntax.emptyToken(kind, fullStart); }