From 161dc656cb9f5757ab517dd03db6f104966e4eb7 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 20 Nov 2014 19:34:48 -0800 Subject: [PATCH] Simplify incremental parsing by moving old source tree nodes before doing anything. --- src/services/syntax/incrementalParser.ts | 430 +++++++++-------------- src/services/syntax/parser.ts | 9 +- src/services/syntax/scanner.ts | 42 +-- src/services/syntax/slidingWindow.ts | 8 + 4 files changed, 203 insertions(+), 286 deletions(-) diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index be0b26cf302..29d3ac07024 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -4,8 +4,10 @@ module TypeScript.IncrementalParser { interface IParserRewindPoint { // Information used by the incremental parser source. oldSourceUnitCursor: SyntaxCursor; - changeDelta: number; - changeRange: TextChangeRange; + } + + interface ISyntaxElementInternal extends ISyntaxToken { + intersectsChange: boolean; } // Parser source used in incremental scenarios. This parser source wraps an old tree, text @@ -31,40 +33,15 @@ module TypeScript.IncrementalParser { // // This parser source also keeps track of the absolute position in the text that we're in, // and any token diagnostics produced. That way we dont' have to track that ourselves. - var _scannerParserSource: Scanner.IScannerParserSource; - - // The range of text in the *original* text that was changed, and the new length of it after - // the change. - var _changeRange: TextChangeRange; - - // Cached value of _changeRange.newSpan(). Cached for performance. - var _changeRangeNewSpan: TextSpan; - - // This number represents how our position in the old tree relates to the position we're - // pointing at in the new text. If it is 0 then our positions are in sync and we can read - // nodes or tokens from the old tree. If it is non-zero, then our positions are not in - // sync and we cannot use nodes or tokens from the old tree. - // - // Now, changeDelta could be negative or positive. Negative means 'the position we're at - // in the original tree is behind the position we're at in the text'. In this case we - // keep throwing out old nodes or tokens (and thus move forward in the original tree) until - // changeDelta becomes 0 again or positive. If it becomes 0 then we are resynched and can - // read nodes or tokesn from the tree. - // - // If changeDelta is positive, that means the current node or token we're pointing at in - // the old tree is at a further ahead position than the position we're pointing at in the - // new text. In this case we have no choice but to scan tokens from teh new text. We will - // continue to do so until, again, changeDelta becomes 0 and we've resynced, or change delta - // becomes negative and we need to skip nodes or tokes in the original tree. - var _changeDelta: number = 0; + var _scannerParserSource = Scanner.createParserSource(oldSyntaxTree.fileName(), text, oldSyntaxTree.languageVersion()); // The cursor we use to navigate through and retrieve nodes and tokens from the old tree. - var _oldSourceUnitCursor = getSyntaxCursor(); var oldSourceUnit = oldSyntaxTree.sourceUnit(); var _outstandingRewindPointCount = 0; // Start the cursor pointing at the first element in the source unit (if it exists). + var _oldSourceUnitCursor = getSyntaxCursor(); if (oldSourceUnit.moduleElements.length > 0) { _oldSourceUnitCursor.pushElement(childAt(oldSourceUnit.moduleElements, 0), /*indexInParent:*/ 0); } @@ -74,8 +51,10 @@ module TypeScript.IncrementalParser { // time this could be problematic would be if the user made a ton of discontinuous edits. // For example, doing a column select on a *large* section of a code. If this is a // problem, we can always update this code to handle multiple changes. - _changeRange = extendToAffectedRange(textChangeRange, oldSourceUnit); - _changeRangeNewSpan = _changeRange.newSpan(); + var _changeRange = extendToAffectedRange(textChangeRange, oldSourceUnit); + + // Cached value of _changeRange.newSpan(). Cached for performance. + var _changeRangeNewSpan = _changeRange.newSpan(); // The old tree's length, plus whatever length change was caused by the edit // Had better equal the new text's length! @@ -83,8 +62,25 @@ module TypeScript.IncrementalParser { Debug.assert((fullWidth(oldSourceUnit) - _changeRange.span().length() + _changeRange.newLength()) === text.length()); } - // Set up a scanner so that we can scan tokens out of the new text. - _scannerParserSource = Scanner.createParserSource(oldSyntaxTree.fileName(), text, oldSyntaxTree.languageVersion()); + var delta = _changeRange.newSpan().length() - _changeRange.span().length(); + if (delta !== 0) { + // If we added or removed characters during the edit, then we need to go and adjust all + // the nodes after the edit. Those nodes may move forward down (if we inserted chars) + // or they may move backward (if we deleted chars). + // + // Doing this helps us out in two ways. First, it means that any nodes/tokens we want + // to reuse are already at the appropriate position in the new text. That way when we + // reuse them, we don't have to figure out if they need to be adjusted. Second, it makes + // it very easy to determine if we can reuse a node. If the node's position is at where + // we are in the text, then we can reuse it. Otherwise we can't. If hte node's position + // is ahead of us, then we'll need to rescan tokens. If the node's position is behind + // us, then we'll need to skip it or crumble it as appropriate + // + // Also, mark any syntax elements that intersect the changed span. We know, up front, + // that we cannot reuse these elements. + updateTokenPositionsAndMarkElements(oldSourceUnit, + _changeRange.span().start(), _changeRange.span().end(),delta); + } function release() { _scannerParserSource.release(); @@ -93,8 +89,7 @@ module TypeScript.IncrementalParser { _outstandingRewindPointCount = 0; } - function extendToAffectedRange(changeRange: TextChangeRange, - sourceUnit: SourceUnitSyntax): TextChangeRange { + function extendToAffectedRange(changeRange: TextChangeRange, sourceUnit: SourceUnitSyntax): TextChangeRange { // Consider the following code: // void foo() { /; } // @@ -105,14 +100,6 @@ module TypeScript.IncrementalParser { // (as it does not intersect the actual original change range). Because an edit may // change the token touching it, we actually need to look back *at least* one token so // that the prior token sees that change. - // - // Note: i believe (outside of regex tokens) max lookahead is just one token for - // TypeScript. However, if this turns out to be wrong, we may have to increase how much - // futher we look back. - // - // Note: lookahead handling for regex characters is handled specially in during - // incremental parsing, and does not need to be handled here. - var maxLookahead = 1; var start = changeRange.span().start(); @@ -149,33 +136,27 @@ module TypeScript.IncrementalParser { // Get a rewind point for our new text reader and for our old source unit cursor. var rewindPoint = _scannerParserSource.getRewindPoint(); - // Clone our cursor. That way we can restore to that point if hte parser needs to rewind. + // Clone our cursor. That way we can restore to that point if the parser needs to rewind. var oldSourceUnitCursorClone = cloneSyntaxCursor(_oldSourceUnitCursor); // Store where we were when the rewind point was created. - rewindPoint.changeDelta = _changeDelta; - rewindPoint.changeRange = _changeRange; rewindPoint.oldSourceUnitCursor = _oldSourceUnitCursor; _oldSourceUnitCursor = oldSourceUnitCursorClone; - // Debug.assert(rewindPoint.pinCount === _oldSourceUnitCursor.pinCount()); - _outstandingRewindPointCount++; return rewindPoint; } function rewind(rewindPoint: IParserRewindPoint): void { // Restore our state to the values when the rewind point was created. - _changeRange = rewindPoint.changeRange; - _changeDelta = rewindPoint.changeDelta; // Reset the cursor to what it was when we got the rewind point. Make sure to return // our existing cursor to the pool so it can be reused. returnSyntaxCursor(_oldSourceUnitCursor); _oldSourceUnitCursor = rewindPoint.oldSourceUnitCursor; - // Null out the cursor that the rewind point points to. This way we don't try + // Clear the cursor that the rewind point points to. This way we don't try // to return it in 'releaseRewindPoint'. rewindPoint.oldSourceUnitCursor = undefined; @@ -220,69 +201,23 @@ module TypeScript.IncrementalParser { // If our current absolute position is in the middle of the changed range in the new text // then we definitely can't read from the old source unit right now. - if (_changeRange && _changeRangeNewSpan.intersectsWithPosition(absolutePosition())) { + var absolutePos = absolutePosition(); + if (_changeRangeNewSpan.intersectsWithPosition(absolutePos)) { return false; } // First, try to sync up with the new text if we're behind. syncCursorToNewTextIfBehind(); - // Now, if we're synced up *and* we're not currently pinned in the new text scanner, - // then we can read a node from the cursor. If we're pinned in the scanner then we - // can't read a node from the cursor because we will mess up the pinned scanner when - // we try to move it forward past this node. - return _changeDelta === 0 && - !_oldSourceUnitCursor.isFinished(); - } + // Now see what the position is of the node we're considering reusing. If it's at our + // current position then we can *potentialy* reuse it. We'll still need to do more + // viability checks later on. This will happen in tryGetNodeFromOldSourceUnit or + // tryGetTokenFromOldSourceUnit. + var currentNodeOrToken = _oldSourceUnitCursor.currentNodeOrToken(); - function updateTokenPosition(token: ISyntaxToken): void { - // If we got a node or token, and we're past the range of edited text, then walk its - // constituent tokens, making sure all their positions are correct. We don't need to - // do this for the tokens before the edited range (since their positions couldn't have - // been affected by the edit), and we don't need to do this for the tokens in the - // edited range, as their positions will be correct when the underlying parser source - // creates them. - - if (isPastChangeRange()) { - token.setFullStart(absolutePosition()); - } - } - - function updateNodePosition(node: ISyntaxNode): void { - // If we got a node or token, and we're past the range of edited text, then walk its - // constituent tokens, making sure all their positions are correct. We don't need to - // do this for the tokens before the edited range (since their positions couldn't have - // been affected by the edit), and we don't need to do this for the tokens in the - // edited range, as their positions will be correct when the underlying parser source - // creates them. - - if (isPastChangeRange()) { - var position = absolutePosition(); - - var tokens = getTokens(node); - - for (var i = 0, n = tokens.length; i < n; i++) { - var token = tokens[i]; - token.setFullStart(position); - - position += token.fullWidth(); - } - } - } - - function getTokens(node: ISyntaxNode): ISyntaxToken[] { - var tokens = node.__cachedTokens; - if (!tokens) { - tokens = []; - tokenCollectorWalker.tokens = tokens; - - visitNodeOrToken(tokenCollectorWalker, node); - - node.__cachedTokens = tokens; - tokenCollectorWalker.tokens = undefined; - } - - return tokens; + // we should never return a node or token that intersects the change range. + Debug.assert(currentNodeOrToken === undefined || !(currentNodeOrToken).intersectsChange); + return currentNodeOrToken && fullStart(currentNodeOrToken) === absolutePos; } function currentNode(): ISyntaxNode { @@ -291,8 +226,6 @@ module TypeScript.IncrementalParser { // to get a token. var node = tryGetNodeFromOldSourceUnit(); if (node) { - // Make sure the positions for the tokens in this node are correct. - updateNodePosition(node); return node; } } @@ -305,8 +238,6 @@ module TypeScript.IncrementalParser { if (canReadFromOldSourceUnit()) { var token = tryGetTokenFromOldSourceUnit(); if (token) { - // Make sure the token's position/text is correct. - updateTokenPosition(token); return token; } } @@ -325,53 +256,58 @@ module TypeScript.IncrementalParser { while (true) { if (_oldSourceUnitCursor.isFinished()) { // Can't sync up if the cursor is finished. - break; + return; } - if (_changeDelta >= 0) { - // Nothing to do if we're synced up or ahead of the text. - break; - } - - // We're behind in the original tree. Throw out a node or token in an attempt to - // catch up to the position we're at in the new text. - + // Start with the current node or token the cursor is pointing at. var currentNodeOrToken = _oldSourceUnitCursor.currentNodeOrToken(); - // If we're pointing at a node, and that node's width is less than our delta, - // then we can just skip that node. Otherwise, if we're pointing at a node - // whose width is greater than the delta, then crumble it and try again. - // Otherwise, we must be pointing at a token. Just skip it and try again. + // Node, move the cursor past any nodes or tokens that intersect the change range + // 1) they are never reusable. + // 2) their positions are wacky as they refer to the original text. + // + // We consider these nodes and tokens essentially invisible to all further parts + // of the incremental algorithm. + if ((currentNodeOrToken).intersectsChange) { + if (isNode(currentNodeOrToken)) { + _oldSourceUnitCursor.moveToFirstChild(); + } + else { + _oldSourceUnitCursor.moveToNextSibling(); + } + continue; + } - if (isNode(currentNodeOrToken) && (fullWidth(currentNodeOrToken) > Math.abs(_changeDelta))) { - // We were pointing at a node whose width was more than changeDelta. Crumble the - // node and try again. Note: we haven't changed changeDelta. So the callers loop - // will just repeat this until we get to a node or token that we can skip over. - _oldSourceUnitCursor.moveToFirstChild(); + var absolutePos = absolutePosition(); + + var currentNodeOrTokenFullStart = fullStart(currentNodeOrToken); + if (currentNodeOrTokenFullStart >= absolutePos) { + // The node or token is either where we're at, or ahead of where we're at. If + // it's where we're at, there's no work to be done. If it's ahead of where + // we're at, we need to rescan tokens until we catch up. in either case, + // we don't need to sync the cursor. + return; + } + + // The node or is behind the current position we're at in the text. + + var currentNodeOrTokenFullWidth = fullWidth(currentNodeOrToken); + var currentNodeOrTokenFullEnd = currentNodeOrTokenFullStart + currentNodeOrTokenFullWidth; + + // If we're pointing at a node, and that node ends before our current position, we + // can just skip the node entirely. Or, if we're pointing at a token, we won't be + // able to break up that token any further and we should just move to the next + // token. + if (currentNodeOrTokenFullEnd <= absolutePos || isToken(currentNodeOrToken)) { + _oldSourceUnitCursor.moveToNextSibling(); } else { - _oldSourceUnitCursor.moveToNextSibling(); - - // Get our change delta closer to 0 as we skip past this item. - _changeDelta += fullWidth(currentNodeOrToken); - - // If this was a node, then our changeDelta is 0 or negative. If this was a - // token, then we could still be negative (and we have to read another token), - // we could be zero (we're done), or we could be positive (we've moved ahead - // of the new text). Only if we're negative will we continue looping. + // We have a node, and it started before our absolute pos, and ended after our + // pos. Try to crumble this node to see if we'll be able to skip the first node + // or token contained within. + _oldSourceUnitCursor.moveToFirstChild(); } } - - // At this point, we must be either: - // a) done with the cursor - // b) (ideally) caught up to the new text position. - // c) ahead of the new text position. - // In case 'b' we can try to reuse a node from teh old tree. - // Debug.assert(_oldSourceUnitCursor.isFinished() || _changeDelta >= 0); - } - - function intersectsWithChangeRangeSpanInOriginalText(start: number, length: number) { - return !isPastChangeRange() && _changeRange.span().intersectsWith(start, length); } function tryGetNodeFromOldSourceUnit(): ISyntaxNode { @@ -379,11 +315,10 @@ module TypeScript.IncrementalParser { // Keep moving the cursor down to the first node that is safe to return. A node is // safe to return if: - // a) it does not intersect the changed text. - // b) it does not contain skipped text. - // c) it does not have any zero width tokens in it. - // d) it does not have a regex token in it. - // e) we are still in the same strict or non-strict state that the node was originally parsed in. + // a) it does not contain skipped text. + // b) it does not have any zero width tokens in it. + // c) it does not have a regex token in it. + // d) we are still in the same strict or non-strict state that the node was originally parsed in. while (true) { var node = _oldSourceUnitCursor.currentNode(); if (node === undefined) { @@ -391,15 +326,12 @@ module TypeScript.IncrementalParser { return undefined; } - if (!intersectsWithChangeRangeSpanInOriginalText(absolutePosition(), fullWidth(node))) { - // Didn't intersect with the change range. - var isIncrementallyUnusuable = TypeScript.isIncrementallyUnusable(node); - if (!isIncrementallyUnusuable) { + var isIncrementallyUnusuable = TypeScript.isIncrementallyUnusable(node); + if (!isIncrementallyUnusuable) { - // Didn't contain anything that would make it unusable. Awesome. This is - // a node we can reuse. - return node; - } + // Didn't contain anything that would make it unusable. Awesome. This is + // a node we can reuse. + return node; } // We couldn't use currentNode. Try to move to its first child (in case that's a @@ -411,10 +343,9 @@ module TypeScript.IncrementalParser { function canReuseTokenFromOldSourceUnit(position: number, token: ISyntaxToken): boolean { // A token is safe to return if: - // a) it does not intersect the changed text. - // b) it does not contain skipped text. - // c) it is not zero width. - // d) it is not a contextual parser token. + // a) it does not contain skipped text. + // b) it is not zero width. + // c) it is not a contextual parser token. // // NOTE: It is safe to get a token regardless of what our strict context was/is. That's // because the strict context doesn't change what tokens are scanned, only how the @@ -428,14 +359,12 @@ module TypeScript.IncrementalParser { // Converted identifiers can't ever be created by the scanner, and as such, should not // be returned by this source. if (token) { - if (!intersectsWithChangeRangeSpanInOriginalText(position, token.fullWidth())) { - // Didn't intersect with the change range. - if (!token.isIncrementallyUnusable() && !Scanner.isContextualToken(token)) { + // Didn't intersect with the change range. + if (!token.isIncrementallyUnusable() && !Scanner.isContextualToken(token)) { - // Didn't contain anything that would make it unusable. Awesome. This is - // a token we can reuse. - return true; - } + // Didn't contain anything that would make it unusable. Awesome. This is + // a token we can reuse. + return true; } } @@ -506,86 +435,8 @@ module TypeScript.IncrementalParser { ? token : undefined; } - function consumeNode(node: ISyntaxNode): void { - // A node could have only come from the old source unit cursor. Update it and our - // current state. - // Debug.assert(_changeDelta === 0); - // Debug.assert(currentNode() === node); - - _oldSourceUnitCursor.moveToNextSibling(); - - // Update the underlying source with where it should now be currently pointin. - var _absolutePosition = absolutePosition() + fullWidth(node); - _scannerParserSource.resetToPosition(_absolutePosition); - - // Debug.assert(previousToken !== undefined); - // Debug.assert(previousToken.width() > 0); - - //if (!isPastChangeRange()) { - // // If we still have a change range, then this node must have ended before the - // // change range starts. Thus, we don't need to call 'skipPastChanges'. - // Debug.assert(absolutePosition() < _changeRange.span().start()); - //} - } - - function consumeToken(currentToken: ISyntaxToken): void { - // Debug.assert(currentToken.fullWidth() > 0 || currentToken.kind === SyntaxKind.EndOfFileToken); - - // This token may have come from the old source unit, or from the new text. Handle - // both accordingly. - - if (_oldSourceUnitCursor.currentToken() === currentToken) { - // The token came from the old source unit. So our tree and text must be in sync. - // Debug.assert(_changeDelta === 0); - - // Move the cursor past this token. - _oldSourceUnitCursor.moveToNextSibling(); - - // Debug.assert(!_normalParserSource.isPinned()); - - // Update the underlying source with where it should now be currently pointing. We - // don't need to do this when the token came from the new text as the source will - // automatically be placed in the right position. - var _absolutePosition = absolutePosition() + currentToken.fullWidth(); - _scannerParserSource.resetToPosition(_absolutePosition); - - // Debug.assert(previousToken !== undefined); - // Debug.assert(previousToken.width() > 0); - - //if (!isPastChangeRange()) { - // // If we still have a change range, then this token must have ended before the - // // change range starts. Thus, we don't need to call 'skipPastChanges'. - // Debug.assert(absolutePosition() < _changeRange.span().start()); - //} - } - else { - // the token came from the new text. That means the normal source moved forward, - // while the syntax cursor stayed in the same place. Thus our delta moves even - // further back. - _changeDelta -= currentToken.fullWidth(); - - // Move our underlying source forward. - _scannerParserSource.consumeToken(currentToken); - - // Because we read a token from the new text, we may have moved ourselves past the - // change range. If we did, then we may also have to update our change delta to - // compensate for the length change between the old and new text. - if (!isPastChangeRange()) { - // var changeEndInNewText = _changeRange.span().start() + _changeRange.newLength(); - if (absolutePosition() >= _changeRangeNewSpan.end()) { - _changeDelta += _changeRange.newLength() - _changeRange.span().length(); - - // Once we're past the change range, we no longer need it. Null it out. - // From now on we can check if we're past the change range just by seeing - // if this is undefined. - _changeRange = undefined; - } - } - } - } - - function isPastChangeRange(): boolean { - return _changeRange === undefined; + function consumeNodeOrToken(nodeOrToken: ISyntaxNodeOrToken): void { + _scannerParserSource.consumeNodeOrToken(nodeOrToken); } return { @@ -597,8 +448,7 @@ module TypeScript.IncrementalParser { currentToken: currentToken, currentContextualToken: currentContextualToken, peekToken: peekToken, - consumeNode: consumeNode, - consumeToken: consumeToken, + consumeNodeOrToken: consumeNodeOrToken, getRewindPoint: getRewindPoint, rewind: rewind, releaseRewindPoint: releaseRewindPoint, @@ -888,6 +738,76 @@ module TypeScript.IncrementalParser { } var tokenCollectorWalker = new TokenCollectorWalker(); + function updateTokenPositionsAndMarkElements(element: ISyntaxElementInternal, changeStart: number, changeRangeOldEnd: number, delta: number): void { + if (element) { + // First, try to skip past any elements that we dont' need to move. We don't need to + // move any elements that don't start on or after the end of the change range. + if (fullStart(element) >= changeRangeOldEnd) { + // This element started on or after the end of the edit. We need to move it. + forceUpdateTokenPositionsForElement(element, delta); + } + else { + // Check if the element intersects the change range. If it does, then it is not + // reusable. Also, we'll need to recurse to see what constituent portions we may + // be able to use. + if (fullEnd(element) >= changeStart) { + element.intersectsChange = true; + + for (var i = 0, n = childCount(element); i < n; i++) { + updateTokenPositionsAndMarkElements(childAt(element, i), changeStart, changeRangeOldEnd, delta); + } + } + // else { + // This element was totally before the edited range. We don't need to do anything + // with it. + // } + } + } + } + + function forceUpdateTokenPositionsForElement(element: ISyntaxElement, delta: number) { + if (isList(element)) { + var list = element; + for (var i = 0, n = list.length; i < n; i++) { + forceUpdateTokenPositionForNodeOrToken(list[i], delta); + } + } + else { + forceUpdateTokenPositionForNodeOrToken(element, delta); + } + } + + function forceUpdateTokenPosition(token: ISyntaxToken, delta: number) { + token.setFullStart(token.fullStart() + delta); + } + + function forceUpdateTokenPositionForNodeOrToken(nodeOrToken: ISyntaxNodeOrToken, delta: number) { + if (isToken(nodeOrToken)) { + forceUpdateTokenPosition(nodeOrToken, delta); + } + else { + var node = nodeOrToken; + var tokens = getTokens(node); + for (var i = 0, n = tokens.length; i < n; i++) { + forceUpdateTokenPosition(tokens[i], delta); + } + } + } + + function getTokens(node: ISyntaxNode): ISyntaxToken[] { + var tokens = node.__cachedTokens; + if (!tokens) { + tokens = []; + tokenCollectorWalker.tokens = tokens; + + visitNodeOrToken(tokenCollectorWalker, node); + + node.__cachedTokens = tokens; + tokenCollectorWalker.tokens = undefined; + } + + return tokens; + } export function parse(oldSyntaxTree: SyntaxTree, textChangeRange: TextChangeRange, newText: ISimpleText): SyntaxTree { if (textChangeRange.isUnchanged()) { diff --git a/src/services/syntax/parser.ts b/src/services/syntax/parser.ts index b1b197cd1ea..764963450cb 100644 --- a/src/services/syntax/parser.ts +++ b/src/services/syntax/parser.ts @@ -70,8 +70,7 @@ module TypeScript.Parser { // Called to move the source to the next node or token once the parser has consumed the // current one. - consumeNode(node: ISyntaxNode): void; - consumeToken(token: ISyntaxToken): void; + consumeNodeOrToken(node: ISyntaxNodeOrToken): void; // Gets a rewind point that the parser can use to move back to after it speculatively // parses something. The source guarantees that if the parser calls 'rewind' with that @@ -320,14 +319,14 @@ module TypeScript.Parser { // call 'consumeToken'. Doing so would attempt to add any previous skipped tokens // to this token we're skipping. We don't want to do that. Instead, we want to add // all the skipped tokens when we finally eat the next good token. - source.consumeToken(token) + source.consumeNodeOrToken(token) } function consumeToken(token: ISyntaxToken): ISyntaxToken { // Debug.assert(token.fullWidth() > 0 || token.kind === SyntaxKind.EndOfFileToken); // First, tell our source that the token has been consumed. - source.consumeToken(token); + source.consumeNodeOrToken(token); // Now, if we had any skipped tokens, we want to add them to the start of this token // we're consuming. @@ -385,7 +384,7 @@ module TypeScript.Parser { function consumeNode(node: ISyntaxNode): void { Debug.assert(_skippedTokens === undefined); - source.consumeNode(node); + source.consumeNodeOrToken(node); } //this method is called very frequently diff --git a/src/services/syntax/scanner.ts b/src/services/syntax/scanner.ts index 17b127eb6a9..14c73f2c6b4 100644 --- a/src/services/syntax/scanner.ts +++ b/src/services/syntax/scanner.ts @@ -1430,16 +1430,6 @@ module TypeScript.Scanner { return !hadError && SyntaxFacts.isIdentifierNameOrAnyKeyword(token) && width(token) === text.length(); } - // A parser source that gets its data from an underlying scanner. - export interface IScannerParserSource extends Parser.IParserSource { - // The position that the scanner is currently at. - absolutePosition(): number; - - // Resets the source to this position. Any diagnostics produced after this point will be - // removed. - resetToPosition(absolutePosition: number): void; - } - interface IScannerRewindPoint extends Parser.IRewindPoint { // Information used by normal parser source. absolutePosition: number; @@ -1449,7 +1439,7 @@ module TypeScript.Scanner { // Parser source used in batch scenarios. Directly calls into an underlying text scanner and // supports none of the functionality to reuse nodes. Good for when you just want want to do // a single parse of a file. - export function createParserSource(fileName: string, text: ISimpleText, languageVersion: ts.ScriptTarget): IScannerParserSource { + export function createParserSource(fileName: string, text: ISimpleText, languageVersion: ts.ScriptTarget): Parser.IParserSource { // The absolute position we're at in the text we're reading from. var _absolutePosition: number = 0; @@ -1489,11 +1479,6 @@ module TypeScript.Scanner { return undefined; } - function consumeNode(node: ISyntaxNode): void { - // Should never get called. - throw Errors.invalidOperation(); - } - function absolutePosition() { return _absolutePosition; } @@ -1561,13 +1546,20 @@ module TypeScript.Scanner { return slidingWindow.peekItemN(n); } - function consumeToken(token: ISyntaxToken): void { - // Debug.assert(token.fullWidth() > 0 || token.kind === SyntaxKind.EndOfFileToken); - - // Debug.assert(currentToken() === token); - _absolutePosition += token.fullWidth(); - - slidingWindow.moveToNextItem(); + function consumeNodeOrToken(nodeOrToken: ISyntaxNodeOrToken): void { + if (nodeOrToken === slidingWindow.currentItemWithoutFetching()) { + // We're consuming the token that was just fetched from us by the parser. We just + // need to move ourselves forward and ditch this token from the sliding window. + _absolutePosition += (nodeOrToken).fullWidth(); + slidingWindow.moveToNextItem(); + } + else { + // We're either consuming a node, or we're consuming a token that wasn't from our + // sliding window. Both cases happen in incremental scenarios when the incremental + // parser uses a node or token from an older tree. In that case, we simply want to + // point ourselves at the end of the element that the parser just consumed. + resetToPosition(fullEnd(nodeOrToken)); + } } function currentToken(): ISyntaxToken { @@ -1644,15 +1636,13 @@ module TypeScript.Scanner { currentToken: currentToken, currentContextualToken: currentContextualToken, peekToken: peekToken, - consumeNode: consumeNode, - consumeToken: consumeToken, + consumeNodeOrToken: consumeNodeOrToken, getRewindPoint: getRewindPoint, rewind: rewind, releaseRewindPoint: releaseRewindPoint, tokenDiagnostics: tokenDiagnostics, release: release, absolutePosition: absolutePosition, - resetToPosition: resetToPosition, }; } diff --git a/src/services/syntax/slidingWindow.ts b/src/services/syntax/slidingWindow.ts index 4bb79023f2f..39502be24f3 100644 --- a/src/services/syntax/slidingWindow.ts +++ b/src/services/syntax/slidingWindow.ts @@ -163,6 +163,14 @@ module TypeScript { return this.window[this.currentRelativeItemIndex]; } + public currentItemWithoutFetching(): any { + if (this.currentRelativeItemIndex >= this.windowCount) { + return undefined; + } + + return this.window[this.currentRelativeItemIndex]; + } + public peekItemN(n: number): any { // Assert disabled because it is actually expensive enugh to affect perf. // Debug.assert(n >= 0);