From 161dc656cb9f5757ab517dd03db6f104966e4eb7 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 20 Nov 2014 19:34:48 -0800 Subject: [PATCH 01/12] Simplify incremental parsing by moving old source tree nodes before doing anything. --- src/services/syntax/incrementalParser.ts | 430 +++++++++-------------- src/services/syntax/parser.ts | 9 +- src/services/syntax/scanner.ts | 42 +-- src/services/syntax/slidingWindow.ts | 8 + 4 files changed, 203 insertions(+), 286 deletions(-) diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index be0b26cf302..29d3ac07024 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -4,8 +4,10 @@ module TypeScript.IncrementalParser { interface IParserRewindPoint { // Information used by the incremental parser source. oldSourceUnitCursor: SyntaxCursor; - changeDelta: number; - changeRange: TextChangeRange; + } + + interface ISyntaxElementInternal extends ISyntaxToken { + intersectsChange: boolean; } // Parser source used in incremental scenarios. This parser source wraps an old tree, text @@ -31,40 +33,15 @@ module TypeScript.IncrementalParser { // // This parser source also keeps track of the absolute position in the text that we're in, // and any token diagnostics produced. That way we dont' have to track that ourselves. - var _scannerParserSource: Scanner.IScannerParserSource; - - // The range of text in the *original* text that was changed, and the new length of it after - // the change. - var _changeRange: TextChangeRange; - - // Cached value of _changeRange.newSpan(). Cached for performance. - var _changeRangeNewSpan: TextSpan; - - // This number represents how our position in the old tree relates to the position we're - // pointing at in the new text. If it is 0 then our positions are in sync and we can read - // nodes or tokens from the old tree. If it is non-zero, then our positions are not in - // sync and we cannot use nodes or tokens from the old tree. - // - // Now, changeDelta could be negative or positive. Negative means 'the position we're at - // in the original tree is behind the position we're at in the text'. In this case we - // keep throwing out old nodes or tokens (and thus move forward in the original tree) until - // changeDelta becomes 0 again or positive. If it becomes 0 then we are resynched and can - // read nodes or tokesn from the tree. - // - // If changeDelta is positive, that means the current node or token we're pointing at in - // the old tree is at a further ahead position than the position we're pointing at in the - // new text. In this case we have no choice but to scan tokens from teh new text. We will - // continue to do so until, again, changeDelta becomes 0 and we've resynced, or change delta - // becomes negative and we need to skip nodes or tokes in the original tree. - var _changeDelta: number = 0; + var _scannerParserSource = Scanner.createParserSource(oldSyntaxTree.fileName(), text, oldSyntaxTree.languageVersion()); // The cursor we use to navigate through and retrieve nodes and tokens from the old tree. - var _oldSourceUnitCursor = getSyntaxCursor(); var oldSourceUnit = oldSyntaxTree.sourceUnit(); var _outstandingRewindPointCount = 0; // Start the cursor pointing at the first element in the source unit (if it exists). + var _oldSourceUnitCursor = getSyntaxCursor(); if (oldSourceUnit.moduleElements.length > 0) { _oldSourceUnitCursor.pushElement(childAt(oldSourceUnit.moduleElements, 0), /*indexInParent:*/ 0); } @@ -74,8 +51,10 @@ module TypeScript.IncrementalParser { // time this could be problematic would be if the user made a ton of discontinuous edits. // For example, doing a column select on a *large* section of a code. If this is a // problem, we can always update this code to handle multiple changes. - _changeRange = extendToAffectedRange(textChangeRange, oldSourceUnit); - _changeRangeNewSpan = _changeRange.newSpan(); + var _changeRange = extendToAffectedRange(textChangeRange, oldSourceUnit); + + // Cached value of _changeRange.newSpan(). Cached for performance. + var _changeRangeNewSpan = _changeRange.newSpan(); // The old tree's length, plus whatever length change was caused by the edit // Had better equal the new text's length! @@ -83,8 +62,25 @@ module TypeScript.IncrementalParser { Debug.assert((fullWidth(oldSourceUnit) - _changeRange.span().length() + _changeRange.newLength()) === text.length()); } - // Set up a scanner so that we can scan tokens out of the new text. - _scannerParserSource = Scanner.createParserSource(oldSyntaxTree.fileName(), text, oldSyntaxTree.languageVersion()); + var delta = _changeRange.newSpan().length() - _changeRange.span().length(); + if (delta !== 0) { + // If we added or removed characters during the edit, then we need to go and adjust all + // the nodes after the edit. Those nodes may move forward down (if we inserted chars) + // or they may move backward (if we deleted chars). + // + // Doing this helps us out in two ways. First, it means that any nodes/tokens we want + // to reuse are already at the appropriate position in the new text. That way when we + // reuse them, we don't have to figure out if they need to be adjusted. Second, it makes + // it very easy to determine if we can reuse a node. If the node's position is at where + // we are in the text, then we can reuse it. Otherwise we can't. If hte node's position + // is ahead of us, then we'll need to rescan tokens. If the node's position is behind + // us, then we'll need to skip it or crumble it as appropriate + // + // Also, mark any syntax elements that intersect the changed span. We know, up front, + // that we cannot reuse these elements. + updateTokenPositionsAndMarkElements(oldSourceUnit, + _changeRange.span().start(), _changeRange.span().end(),delta); + } function release() { _scannerParserSource.release(); @@ -93,8 +89,7 @@ module TypeScript.IncrementalParser { _outstandingRewindPointCount = 0; } - function extendToAffectedRange(changeRange: TextChangeRange, - sourceUnit: SourceUnitSyntax): TextChangeRange { + function extendToAffectedRange(changeRange: TextChangeRange, sourceUnit: SourceUnitSyntax): TextChangeRange { // Consider the following code: // void foo() { /; } // @@ -105,14 +100,6 @@ module TypeScript.IncrementalParser { // (as it does not intersect the actual original change range). Because an edit may // change the token touching it, we actually need to look back *at least* one token so // that the prior token sees that change. - // - // Note: i believe (outside of regex tokens) max lookahead is just one token for - // TypeScript. However, if this turns out to be wrong, we may have to increase how much - // futher we look back. - // - // Note: lookahead handling for regex characters is handled specially in during - // incremental parsing, and does not need to be handled here. - var maxLookahead = 1; var start = changeRange.span().start(); @@ -149,33 +136,27 @@ module TypeScript.IncrementalParser { // Get a rewind point for our new text reader and for our old source unit cursor. var rewindPoint = _scannerParserSource.getRewindPoint(); - // Clone our cursor. That way we can restore to that point if hte parser needs to rewind. + // Clone our cursor. That way we can restore to that point if the parser needs to rewind. var oldSourceUnitCursorClone = cloneSyntaxCursor(_oldSourceUnitCursor); // Store where we were when the rewind point was created. - rewindPoint.changeDelta = _changeDelta; - rewindPoint.changeRange = _changeRange; rewindPoint.oldSourceUnitCursor = _oldSourceUnitCursor; _oldSourceUnitCursor = oldSourceUnitCursorClone; - // Debug.assert(rewindPoint.pinCount === _oldSourceUnitCursor.pinCount()); - _outstandingRewindPointCount++; return rewindPoint; } function rewind(rewindPoint: IParserRewindPoint): void { // Restore our state to the values when the rewind point was created. - _changeRange = rewindPoint.changeRange; - _changeDelta = rewindPoint.changeDelta; // Reset the cursor to what it was when we got the rewind point. Make sure to return // our existing cursor to the pool so it can be reused. returnSyntaxCursor(_oldSourceUnitCursor); _oldSourceUnitCursor = rewindPoint.oldSourceUnitCursor; - // Null out the cursor that the rewind point points to. This way we don't try + // Clear the cursor that the rewind point points to. This way we don't try // to return it in 'releaseRewindPoint'. rewindPoint.oldSourceUnitCursor = undefined; @@ -220,69 +201,23 @@ module TypeScript.IncrementalParser { // If our current absolute position is in the middle of the changed range in the new text // then we definitely can't read from the old source unit right now. - if (_changeRange && _changeRangeNewSpan.intersectsWithPosition(absolutePosition())) { + var absolutePos = absolutePosition(); + if (_changeRangeNewSpan.intersectsWithPosition(absolutePos)) { return false; } // First, try to sync up with the new text if we're behind. syncCursorToNewTextIfBehind(); - // Now, if we're synced up *and* we're not currently pinned in the new text scanner, - // then we can read a node from the cursor. If we're pinned in the scanner then we - // can't read a node from the cursor because we will mess up the pinned scanner when - // we try to move it forward past this node. - return _changeDelta === 0 && - !_oldSourceUnitCursor.isFinished(); - } + // Now see what the position is of the node we're considering reusing. If it's at our + // current position then we can *potentialy* reuse it. We'll still need to do more + // viability checks later on. This will happen in tryGetNodeFromOldSourceUnit or + // tryGetTokenFromOldSourceUnit. + var currentNodeOrToken = _oldSourceUnitCursor.currentNodeOrToken(); - function updateTokenPosition(token: ISyntaxToken): void { - // If we got a node or token, and we're past the range of edited text, then walk its - // constituent tokens, making sure all their positions are correct. We don't need to - // do this for the tokens before the edited range (since their positions couldn't have - // been affected by the edit), and we don't need to do this for the tokens in the - // edited range, as their positions will be correct when the underlying parser source - // creates them. - - if (isPastChangeRange()) { - token.setFullStart(absolutePosition()); - } - } - - function updateNodePosition(node: ISyntaxNode): void { - // If we got a node or token, and we're past the range of edited text, then walk its - // constituent tokens, making sure all their positions are correct. We don't need to - // do this for the tokens before the edited range (since their positions couldn't have - // been affected by the edit), and we don't need to do this for the tokens in the - // edited range, as their positions will be correct when the underlying parser source - // creates them. - - if (isPastChangeRange()) { - var position = absolutePosition(); - - var tokens = getTokens(node); - - for (var i = 0, n = tokens.length; i < n; i++) { - var token = tokens[i]; - token.setFullStart(position); - - position += token.fullWidth(); - } - } - } - - function getTokens(node: ISyntaxNode): ISyntaxToken[] { - var tokens = node.__cachedTokens; - if (!tokens) { - tokens = []; - tokenCollectorWalker.tokens = tokens; - - visitNodeOrToken(tokenCollectorWalker, node); - - node.__cachedTokens = tokens; - tokenCollectorWalker.tokens = undefined; - } - - return tokens; + // we should never return a node or token that intersects the change range. + Debug.assert(currentNodeOrToken === undefined || !(currentNodeOrToken).intersectsChange); + return currentNodeOrToken && fullStart(currentNodeOrToken) === absolutePos; } function currentNode(): ISyntaxNode { @@ -291,8 +226,6 @@ module TypeScript.IncrementalParser { // to get a token. var node = tryGetNodeFromOldSourceUnit(); if (node) { - // Make sure the positions for the tokens in this node are correct. - updateNodePosition(node); return node; } } @@ -305,8 +238,6 @@ module TypeScript.IncrementalParser { if (canReadFromOldSourceUnit()) { var token = tryGetTokenFromOldSourceUnit(); if (token) { - // Make sure the token's position/text is correct. - updateTokenPosition(token); return token; } } @@ -325,53 +256,58 @@ module TypeScript.IncrementalParser { while (true) { if (_oldSourceUnitCursor.isFinished()) { // Can't sync up if the cursor is finished. - break; + return; } - if (_changeDelta >= 0) { - // Nothing to do if we're synced up or ahead of the text. - break; - } - - // We're behind in the original tree. Throw out a node or token in an attempt to - // catch up to the position we're at in the new text. - + // Start with the current node or token the cursor is pointing at. var currentNodeOrToken = _oldSourceUnitCursor.currentNodeOrToken(); - // If we're pointing at a node, and that node's width is less than our delta, - // then we can just skip that node. Otherwise, if we're pointing at a node - // whose width is greater than the delta, then crumble it and try again. - // Otherwise, we must be pointing at a token. Just skip it and try again. + // Node, move the cursor past any nodes or tokens that intersect the change range + // 1) they are never reusable. + // 2) their positions are wacky as they refer to the original text. + // + // We consider these nodes and tokens essentially invisible to all further parts + // of the incremental algorithm. + if ((currentNodeOrToken).intersectsChange) { + if (isNode(currentNodeOrToken)) { + _oldSourceUnitCursor.moveToFirstChild(); + } + else { + _oldSourceUnitCursor.moveToNextSibling(); + } + continue; + } - if (isNode(currentNodeOrToken) && (fullWidth(currentNodeOrToken) > Math.abs(_changeDelta))) { - // We were pointing at a node whose width was more than changeDelta. Crumble the - // node and try again. Note: we haven't changed changeDelta. So the callers loop - // will just repeat this until we get to a node or token that we can skip over. - _oldSourceUnitCursor.moveToFirstChild(); + var absolutePos = absolutePosition(); + + var currentNodeOrTokenFullStart = fullStart(currentNodeOrToken); + if (currentNodeOrTokenFullStart >= absolutePos) { + // The node or token is either where we're at, or ahead of where we're at. If + // it's where we're at, there's no work to be done. If it's ahead of where + // we're at, we need to rescan tokens until we catch up. in either case, + // we don't need to sync the cursor. + return; + } + + // The node or is behind the current position we're at in the text. + + var currentNodeOrTokenFullWidth = fullWidth(currentNodeOrToken); + var currentNodeOrTokenFullEnd = currentNodeOrTokenFullStart + currentNodeOrTokenFullWidth; + + // If we're pointing at a node, and that node ends before our current position, we + // can just skip the node entirely. Or, if we're pointing at a token, we won't be + // able to break up that token any further and we should just move to the next + // token. + if (currentNodeOrTokenFullEnd <= absolutePos || isToken(currentNodeOrToken)) { + _oldSourceUnitCursor.moveToNextSibling(); } else { - _oldSourceUnitCursor.moveToNextSibling(); - - // Get our change delta closer to 0 as we skip past this item. - _changeDelta += fullWidth(currentNodeOrToken); - - // If this was a node, then our changeDelta is 0 or negative. If this was a - // token, then we could still be negative (and we have to read another token), - // we could be zero (we're done), or we could be positive (we've moved ahead - // of the new text). Only if we're negative will we continue looping. + // We have a node, and it started before our absolute pos, and ended after our + // pos. Try to crumble this node to see if we'll be able to skip the first node + // or token contained within. + _oldSourceUnitCursor.moveToFirstChild(); } } - - // At this point, we must be either: - // a) done with the cursor - // b) (ideally) caught up to the new text position. - // c) ahead of the new text position. - // In case 'b' we can try to reuse a node from teh old tree. - // Debug.assert(_oldSourceUnitCursor.isFinished() || _changeDelta >= 0); - } - - function intersectsWithChangeRangeSpanInOriginalText(start: number, length: number) { - return !isPastChangeRange() && _changeRange.span().intersectsWith(start, length); } function tryGetNodeFromOldSourceUnit(): ISyntaxNode { @@ -379,11 +315,10 @@ module TypeScript.IncrementalParser { // Keep moving the cursor down to the first node that is safe to return. A node is // safe to return if: - // a) it does not intersect the changed text. - // b) it does not contain skipped text. - // c) it does not have any zero width tokens in it. - // d) it does not have a regex token in it. - // e) we are still in the same strict or non-strict state that the node was originally parsed in. + // a) it does not contain skipped text. + // b) it does not have any zero width tokens in it. + // c) it does not have a regex token in it. + // d) we are still in the same strict or non-strict state that the node was originally parsed in. while (true) { var node = _oldSourceUnitCursor.currentNode(); if (node === undefined) { @@ -391,15 +326,12 @@ module TypeScript.IncrementalParser { return undefined; } - if (!intersectsWithChangeRangeSpanInOriginalText(absolutePosition(), fullWidth(node))) { - // Didn't intersect with the change range. - var isIncrementallyUnusuable = TypeScript.isIncrementallyUnusable(node); - if (!isIncrementallyUnusuable) { + var isIncrementallyUnusuable = TypeScript.isIncrementallyUnusable(node); + if (!isIncrementallyUnusuable) { - // Didn't contain anything that would make it unusable. Awesome. This is - // a node we can reuse. - return node; - } + // Didn't contain anything that would make it unusable. Awesome. This is + // a node we can reuse. + return node; } // We couldn't use currentNode. Try to move to its first child (in case that's a @@ -411,10 +343,9 @@ module TypeScript.IncrementalParser { function canReuseTokenFromOldSourceUnit(position: number, token: ISyntaxToken): boolean { // A token is safe to return if: - // a) it does not intersect the changed text. - // b) it does not contain skipped text. - // c) it is not zero width. - // d) it is not a contextual parser token. + // a) it does not contain skipped text. + // b) it is not zero width. + // c) it is not a contextual parser token. // // NOTE: It is safe to get a token regardless of what our strict context was/is. That's // because the strict context doesn't change what tokens are scanned, only how the @@ -428,14 +359,12 @@ module TypeScript.IncrementalParser { // Converted identifiers can't ever be created by the scanner, and as such, should not // be returned by this source. if (token) { - if (!intersectsWithChangeRangeSpanInOriginalText(position, token.fullWidth())) { - // Didn't intersect with the change range. - if (!token.isIncrementallyUnusable() && !Scanner.isContextualToken(token)) { + // Didn't intersect with the change range. + if (!token.isIncrementallyUnusable() && !Scanner.isContextualToken(token)) { - // Didn't contain anything that would make it unusable. Awesome. This is - // a token we can reuse. - return true; - } + // Didn't contain anything that would make it unusable. Awesome. This is + // a token we can reuse. + return true; } } @@ -506,86 +435,8 @@ module TypeScript.IncrementalParser { ? token : undefined; } - function consumeNode(node: ISyntaxNode): void { - // A node could have only come from the old source unit cursor. Update it and our - // current state. - // Debug.assert(_changeDelta === 0); - // Debug.assert(currentNode() === node); - - _oldSourceUnitCursor.moveToNextSibling(); - - // Update the underlying source with where it should now be currently pointin. - var _absolutePosition = absolutePosition() + fullWidth(node); - _scannerParserSource.resetToPosition(_absolutePosition); - - // Debug.assert(previousToken !== undefined); - // Debug.assert(previousToken.width() > 0); - - //if (!isPastChangeRange()) { - // // If we still have a change range, then this node must have ended before the - // // change range starts. Thus, we don't need to call 'skipPastChanges'. - // Debug.assert(absolutePosition() < _changeRange.span().start()); - //} - } - - function consumeToken(currentToken: ISyntaxToken): void { - // Debug.assert(currentToken.fullWidth() > 0 || currentToken.kind === SyntaxKind.EndOfFileToken); - - // This token may have come from the old source unit, or from the new text. Handle - // both accordingly. - - if (_oldSourceUnitCursor.currentToken() === currentToken) { - // The token came from the old source unit. So our tree and text must be in sync. - // Debug.assert(_changeDelta === 0); - - // Move the cursor past this token. - _oldSourceUnitCursor.moveToNextSibling(); - - // Debug.assert(!_normalParserSource.isPinned()); - - // Update the underlying source with where it should now be currently pointing. We - // don't need to do this when the token came from the new text as the source will - // automatically be placed in the right position. - var _absolutePosition = absolutePosition() + currentToken.fullWidth(); - _scannerParserSource.resetToPosition(_absolutePosition); - - // Debug.assert(previousToken !== undefined); - // Debug.assert(previousToken.width() > 0); - - //if (!isPastChangeRange()) { - // // If we still have a change range, then this token must have ended before the - // // change range starts. Thus, we don't need to call 'skipPastChanges'. - // Debug.assert(absolutePosition() < _changeRange.span().start()); - //} - } - else { - // the token came from the new text. That means the normal source moved forward, - // while the syntax cursor stayed in the same place. Thus our delta moves even - // further back. - _changeDelta -= currentToken.fullWidth(); - - // Move our underlying source forward. - _scannerParserSource.consumeToken(currentToken); - - // Because we read a token from the new text, we may have moved ourselves past the - // change range. If we did, then we may also have to update our change delta to - // compensate for the length change between the old and new text. - if (!isPastChangeRange()) { - // var changeEndInNewText = _changeRange.span().start() + _changeRange.newLength(); - if (absolutePosition() >= _changeRangeNewSpan.end()) { - _changeDelta += _changeRange.newLength() - _changeRange.span().length(); - - // Once we're past the change range, we no longer need it. Null it out. - // From now on we can check if we're past the change range just by seeing - // if this is undefined. - _changeRange = undefined; - } - } - } - } - - function isPastChangeRange(): boolean { - return _changeRange === undefined; + function consumeNodeOrToken(nodeOrToken: ISyntaxNodeOrToken): void { + _scannerParserSource.consumeNodeOrToken(nodeOrToken); } return { @@ -597,8 +448,7 @@ module TypeScript.IncrementalParser { currentToken: currentToken, currentContextualToken: currentContextualToken, peekToken: peekToken, - consumeNode: consumeNode, - consumeToken: consumeToken, + consumeNodeOrToken: consumeNodeOrToken, getRewindPoint: getRewindPoint, rewind: rewind, releaseRewindPoint: releaseRewindPoint, @@ -888,6 +738,76 @@ module TypeScript.IncrementalParser { } var tokenCollectorWalker = new TokenCollectorWalker(); + function updateTokenPositionsAndMarkElements(element: ISyntaxElementInternal, changeStart: number, changeRangeOldEnd: number, delta: number): void { + if (element) { + // First, try to skip past any elements that we dont' need to move. We don't need to + // move any elements that don't start on or after the end of the change range. + if (fullStart(element) >= changeRangeOldEnd) { + // This element started on or after the end of the edit. We need to move it. + forceUpdateTokenPositionsForElement(element, delta); + } + else { + // Check if the element intersects the change range. If it does, then it is not + // reusable. Also, we'll need to recurse to see what constituent portions we may + // be able to use. + if (fullEnd(element) >= changeStart) { + element.intersectsChange = true; + + for (var i = 0, n = childCount(element); i < n; i++) { + updateTokenPositionsAndMarkElements(childAt(element, i), changeStart, changeRangeOldEnd, delta); + } + } + // else { + // This element was totally before the edited range. We don't need to do anything + // with it. + // } + } + } + } + + function forceUpdateTokenPositionsForElement(element: ISyntaxElement, delta: number) { + if (isList(element)) { + var list = element; + for (var i = 0, n = list.length; i < n; i++) { + forceUpdateTokenPositionForNodeOrToken(list[i], delta); + } + } + else { + forceUpdateTokenPositionForNodeOrToken(element, delta); + } + } + + function forceUpdateTokenPosition(token: ISyntaxToken, delta: number) { + token.setFullStart(token.fullStart() + delta); + } + + function forceUpdateTokenPositionForNodeOrToken(nodeOrToken: ISyntaxNodeOrToken, delta: number) { + if (isToken(nodeOrToken)) { + forceUpdateTokenPosition(nodeOrToken, delta); + } + else { + var node = nodeOrToken; + var tokens = getTokens(node); + for (var i = 0, n = tokens.length; i < n; i++) { + forceUpdateTokenPosition(tokens[i], delta); + } + } + } + + function getTokens(node: ISyntaxNode): ISyntaxToken[] { + var tokens = node.__cachedTokens; + if (!tokens) { + tokens = []; + tokenCollectorWalker.tokens = tokens; + + visitNodeOrToken(tokenCollectorWalker, node); + + node.__cachedTokens = tokens; + tokenCollectorWalker.tokens = undefined; + } + + return tokens; + } export function parse(oldSyntaxTree: SyntaxTree, textChangeRange: TextChangeRange, newText: ISimpleText): SyntaxTree { if (textChangeRange.isUnchanged()) { diff --git a/src/services/syntax/parser.ts b/src/services/syntax/parser.ts index b1b197cd1ea..764963450cb 100644 --- a/src/services/syntax/parser.ts +++ b/src/services/syntax/parser.ts @@ -70,8 +70,7 @@ module TypeScript.Parser { // Called to move the source to the next node or token once the parser has consumed the // current one. - consumeNode(node: ISyntaxNode): void; - consumeToken(token: ISyntaxToken): void; + consumeNodeOrToken(node: ISyntaxNodeOrToken): void; // Gets a rewind point that the parser can use to move back to after it speculatively // parses something. The source guarantees that if the parser calls 'rewind' with that @@ -320,14 +319,14 @@ module TypeScript.Parser { // call 'consumeToken'. Doing so would attempt to add any previous skipped tokens // to this token we're skipping. We don't want to do that. Instead, we want to add // all the skipped tokens when we finally eat the next good token. - source.consumeToken(token) + source.consumeNodeOrToken(token) } function consumeToken(token: ISyntaxToken): ISyntaxToken { // Debug.assert(token.fullWidth() > 0 || token.kind === SyntaxKind.EndOfFileToken); // First, tell our source that the token has been consumed. - source.consumeToken(token); + source.consumeNodeOrToken(token); // Now, if we had any skipped tokens, we want to add them to the start of this token // we're consuming. @@ -385,7 +384,7 @@ module TypeScript.Parser { function consumeNode(node: ISyntaxNode): void { Debug.assert(_skippedTokens === undefined); - source.consumeNode(node); + source.consumeNodeOrToken(node); } //this method is called very frequently diff --git a/src/services/syntax/scanner.ts b/src/services/syntax/scanner.ts index 17b127eb6a9..14c73f2c6b4 100644 --- a/src/services/syntax/scanner.ts +++ b/src/services/syntax/scanner.ts @@ -1430,16 +1430,6 @@ module TypeScript.Scanner { return !hadError && SyntaxFacts.isIdentifierNameOrAnyKeyword(token) && width(token) === text.length(); } - // A parser source that gets its data from an underlying scanner. - export interface IScannerParserSource extends Parser.IParserSource { - // The position that the scanner is currently at. - absolutePosition(): number; - - // Resets the source to this position. Any diagnostics produced after this point will be - // removed. - resetToPosition(absolutePosition: number): void; - } - interface IScannerRewindPoint extends Parser.IRewindPoint { // Information used by normal parser source. absolutePosition: number; @@ -1449,7 +1439,7 @@ module TypeScript.Scanner { // Parser source used in batch scenarios. Directly calls into an underlying text scanner and // supports none of the functionality to reuse nodes. Good for when you just want want to do // a single parse of a file. - export function createParserSource(fileName: string, text: ISimpleText, languageVersion: ts.ScriptTarget): IScannerParserSource { + export function createParserSource(fileName: string, text: ISimpleText, languageVersion: ts.ScriptTarget): Parser.IParserSource { // The absolute position we're at in the text we're reading from. var _absolutePosition: number = 0; @@ -1489,11 +1479,6 @@ module TypeScript.Scanner { return undefined; } - function consumeNode(node: ISyntaxNode): void { - // Should never get called. - throw Errors.invalidOperation(); - } - function absolutePosition() { return _absolutePosition; } @@ -1561,13 +1546,20 @@ module TypeScript.Scanner { return slidingWindow.peekItemN(n); } - function consumeToken(token: ISyntaxToken): void { - // Debug.assert(token.fullWidth() > 0 || token.kind === SyntaxKind.EndOfFileToken); - - // Debug.assert(currentToken() === token); - _absolutePosition += token.fullWidth(); - - slidingWindow.moveToNextItem(); + function consumeNodeOrToken(nodeOrToken: ISyntaxNodeOrToken): void { + if (nodeOrToken === slidingWindow.currentItemWithoutFetching()) { + // We're consuming the token that was just fetched from us by the parser. We just + // need to move ourselves forward and ditch this token from the sliding window. + _absolutePosition += (nodeOrToken).fullWidth(); + slidingWindow.moveToNextItem(); + } + else { + // We're either consuming a node, or we're consuming a token that wasn't from our + // sliding window. Both cases happen in incremental scenarios when the incremental + // parser uses a node or token from an older tree. In that case, we simply want to + // point ourselves at the end of the element that the parser just consumed. + resetToPosition(fullEnd(nodeOrToken)); + } } function currentToken(): ISyntaxToken { @@ -1644,15 +1636,13 @@ module TypeScript.Scanner { currentToken: currentToken, currentContextualToken: currentContextualToken, peekToken: peekToken, - consumeNode: consumeNode, - consumeToken: consumeToken, + consumeNodeOrToken: consumeNodeOrToken, getRewindPoint: getRewindPoint, rewind: rewind, releaseRewindPoint: releaseRewindPoint, tokenDiagnostics: tokenDiagnostics, release: release, absolutePosition: absolutePosition, - resetToPosition: resetToPosition, }; } diff --git a/src/services/syntax/slidingWindow.ts b/src/services/syntax/slidingWindow.ts index 4bb79023f2f..39502be24f3 100644 --- a/src/services/syntax/slidingWindow.ts +++ b/src/services/syntax/slidingWindow.ts @@ -163,6 +163,14 @@ module TypeScript { return this.window[this.currentRelativeItemIndex]; } + public currentItemWithoutFetching(): any { + if (this.currentRelativeItemIndex >= this.windowCount) { + return undefined; + } + + return this.window[this.currentRelativeItemIndex]; + } + public peekItemN(n: number): any { // Assert disabled because it is actually expensive enugh to affect perf. // Debug.assert(n >= 0); From c6088ce683d5017a4abe779ac13a273838026b91 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 21 Nov 2014 01:28:42 -0800 Subject: [PATCH 02/12] Fix interface. --- src/services/syntax/incrementalParser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index 29d3ac07024..ae4d41346cf 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -6,7 +6,7 @@ module TypeScript.IncrementalParser { oldSourceUnitCursor: SyntaxCursor; } - interface ISyntaxElementInternal extends ISyntaxToken { + interface ISyntaxElementInternal extends ISyntaxElement { intersectsChange: boolean; } From 19198256fb6d7a4483c56fba251c8ad5075ae217 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 21 Nov 2014 01:31:15 -0800 Subject: [PATCH 03/12] Simplify parser initializer. --- src/services/syntax/incrementalParser.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index ae4d41346cf..490d3550817 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -23,9 +23,6 @@ module TypeScript.IncrementalParser { // prevent this level of reuse include substantially destructive operations like introducing // "/*" without a "*/" nearby to terminate the comment. function createParserSource(oldSyntaxTree: SyntaxTree, textChangeRange: TextChangeRange, text: ISimpleText): Parser.IParserSource { - var fileName = oldSyntaxTree.fileName(); - var languageVersion = oldSyntaxTree.languageVersion(); - // The underlying source that we will use to scan tokens from any new text, or any tokens // from the old tree that we decide we can't use for any reason. We will also continue // scanning tokens from this source until we've decided that we're resynchronized and can @@ -441,8 +438,8 @@ module TypeScript.IncrementalParser { return { text: text, - fileName: fileName, - languageVersion: languageVersion, + fileName: oldSyntaxTree.fileName(), + languageVersion: oldSyntaxTree.languageVersion(), absolutePosition: absolutePosition, currentNode: currentNode, currentToken: currentToken, From 6afd5e4ad24c77ef5d07acef033415e0203cb90f Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 21 Nov 2014 02:12:59 -0800 Subject: [PATCH 04/12] Simplify incremental code. --- src/services/syntax/incrementalParser.ts | 31 ++++++------------------ 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index 490d3550817..3d2c33fd5e8 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -338,7 +338,7 @@ module TypeScript.IncrementalParser { } } - function canReuseTokenFromOldSourceUnit(position: number, token: ISyntaxToken): boolean { + function canReuseTokenFromOldSourceUnit(token: ISyntaxToken): boolean { // A token is safe to return if: // a) it does not contain skipped text. // b) it is not zero width. @@ -355,17 +355,10 @@ module TypeScript.IncrementalParser { // need to make sure that if that the parser asks for a *token* we don't return it. // Converted identifiers can't ever be created by the scanner, and as such, should not // be returned by this source. - if (token) { - // Didn't intersect with the change range. - if (!token.isIncrementallyUnusable() && !Scanner.isContextualToken(token)) { - - // Didn't contain anything that would make it unusable. Awesome. This is - // a token we can reuse. - return true; - } - } - - return false; + return token && + !(token).intersectsChange && + !token.isIncrementallyUnusable() && + !Scanner.isContextualToken(token); } function tryGetTokenFromOldSourceUnit(): ISyntaxToken { @@ -374,8 +367,7 @@ module TypeScript.IncrementalParser { // get the current token that the cursor is pointing at. var token = _oldSourceUnitCursor.currentToken(); - return canReuseTokenFromOldSourceUnit(absolutePosition(), token) - ? token : undefined; + return canReuseTokenFromOldSourceUnit(token) ? token : undefined; } function peekToken(n: number): ISyntaxToken { @@ -407,11 +399,6 @@ module TypeScript.IncrementalParser { } function tryPeekTokenFromOldSourceUnitWorker(n: number): ISyntaxToken { - // In order to peek the 'nth' token we need all the tokens up to that point. That way - // we know we know position that the nth token is at. The position is necessary so - // that we can test if this token (or any that precede it cross the change range). - var currentPosition = absolutePosition(); - // First, make sure the cursor is pointing at a token. _oldSourceUnitCursor.moveToFirstToken(); @@ -419,17 +406,15 @@ module TypeScript.IncrementalParser { for (var i = 0; i < n; i++) { var interimToken = _oldSourceUnitCursor.currentToken(); - if (!canReuseTokenFromOldSourceUnit(currentPosition, interimToken)) { + if (!canReuseTokenFromOldSourceUnit(interimToken)) { return undefined; } - currentPosition += interimToken.fullWidth(); _oldSourceUnitCursor.moveToNextSibling(); } var token = _oldSourceUnitCursor.currentToken(); - return canReuseTokenFromOldSourceUnit(currentPosition, token) - ? token : undefined; + return canReuseTokenFromOldSourceUnit(token) ? token : undefined; } function consumeNodeOrToken(nodeOrToken: ISyntaxNodeOrToken): void { From 3c3e3b23ddc3123f4c5c1f1a40f38dee91bc96a2 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 21 Nov 2014 14:24:31 -0800 Subject: [PATCH 05/12] Always mark nodes and tokens that cross the edited range. --- src/services/syntax/incrementalParser.ts | 66 ++++++++++++------------ 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index 3d2c33fd5e8..b086b984926 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -60,24 +60,22 @@ module TypeScript.IncrementalParser { } var delta = _changeRange.newSpan().length() - _changeRange.span().length(); - if (delta !== 0) { - // If we added or removed characters during the edit, then we need to go and adjust all - // the nodes after the edit. Those nodes may move forward down (if we inserted chars) - // or they may move backward (if we deleted chars). - // - // Doing this helps us out in two ways. First, it means that any nodes/tokens we want - // to reuse are already at the appropriate position in the new text. That way when we - // reuse them, we don't have to figure out if they need to be adjusted. Second, it makes - // it very easy to determine if we can reuse a node. If the node's position is at where - // we are in the text, then we can reuse it. Otherwise we can't. If hte node's position - // is ahead of us, then we'll need to rescan tokens. If the node's position is behind - // us, then we'll need to skip it or crumble it as appropriate - // - // Also, mark any syntax elements that intersect the changed span. We know, up front, - // that we cannot reuse these elements. - updateTokenPositionsAndMarkElements(oldSourceUnit, - _changeRange.span().start(), _changeRange.span().end(),delta); - } + // If we added or removed characters during the edit, then we need to go and adjust all + // the nodes after the edit. Those nodes may move forward down (if we inserted chars) + // or they may move backward (if we deleted chars). + // + // Doing this helps us out in two ways. First, it means that any nodes/tokens we want + // to reuse are already at the appropriate position in the new text. That way when we + // reuse them, we don't have to figure out if they need to be adjusted. Second, it makes + // it very easy to determine if we can reuse a node. If the node's position is at where + // we are in the text, then we can reuse it. Otherwise we can't. If hte node's position + // is ahead of us, then we'll need to rescan tokens. If the node's position is behind + // us, then we'll need to skip it or crumble it as appropriate + // + // Also, mark any syntax elements that intersect the changed span. We know, up front, + // that we cannot reuse these elements. + updateTokenPositionsAndMarkElements(oldSourceUnit, + _changeRange.span().start(), _changeRange.span().end(), delta); function release() { _scannerParserSource.release(); @@ -199,9 +197,9 @@ module TypeScript.IncrementalParser { // If our current absolute position is in the middle of the changed range in the new text // then we definitely can't read from the old source unit right now. var absolutePos = absolutePosition(); - if (_changeRangeNewSpan.intersectsWithPosition(absolutePos)) { - return false; - } + //if (_changeRangeNewSpan.intersectsWithPosition(absolutePos)) { + // return false; + //} // First, try to sync up with the new text if we're behind. syncCursorToNewTextIfBehind(); @@ -723,9 +721,10 @@ module TypeScript.IncrementalParser { function updateTokenPositionsAndMarkElements(element: ISyntaxElementInternal, changeStart: number, changeRangeOldEnd: number, delta: number): void { if (element) { // First, try to skip past any elements that we dont' need to move. We don't need to - // move any elements that don't start on or after the end of the change range. - if (fullStart(element) >= changeRangeOldEnd) { - // This element started on or after the end of the edit. We need to move it. + // move any elements that don't start after the end of the change range. + if (fullStart(element) > changeRangeOldEnd) { + // Note, we only move elements that are truly after the end of the change range. + // We consider elements that are touching the end of the change range to be unusable. forceUpdateTokenPositionsForElement(element, delta); } else { @@ -740,7 +739,7 @@ module TypeScript.IncrementalParser { } } // else { - // This element was totally before the edited range. We don't need to do anything + // This element ended strictly before the edited range. We don't need to do anything // with it. // } } @@ -748,14 +747,17 @@ module TypeScript.IncrementalParser { } function forceUpdateTokenPositionsForElement(element: ISyntaxElement, delta: number) { - if (isList(element)) { - var list = element; - for (var i = 0, n = list.length; i < n; i++) { - forceUpdateTokenPositionForNodeOrToken(list[i], delta); + // No need to move anything if the delta is 0. + if (delta !== 0) { + if (isList(element)) { + var list = element; + for (var i = 0, n = list.length; i < n; i++) { + forceUpdateTokenPositionForNodeOrToken(list[i], delta); + } + } + else { + forceUpdateTokenPositionForNodeOrToken(element, delta); } - } - else { - forceUpdateTokenPositionForNodeOrToken(element, delta); } } From 45c3dffd4e257b3e3191072374bea71e831726a9 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 21 Nov 2014 14:39:06 -0800 Subject: [PATCH 06/12] Slightly speed up marking by avoiding calling fullStart on so many nodes and tokens. --- src/services/syntax/incrementalParser.ts | 43 +++++++++++++----------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index b086b984926..8af373ce5da 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -75,7 +75,7 @@ module TypeScript.IncrementalParser { // Also, mark any syntax elements that intersect the changed span. We know, up front, // that we cannot reuse these elements. updateTokenPositionsAndMarkElements(oldSourceUnit, - _changeRange.span().start(), _changeRange.span().end(), delta); + _changeRange.span().start(), _changeRange.span().end(), delta, /*fullStart:*/ 0); function release() { _scannerParserSource.release(); @@ -718,31 +718,34 @@ module TypeScript.IncrementalParser { } var tokenCollectorWalker = new TokenCollectorWalker(); - function updateTokenPositionsAndMarkElements(element: ISyntaxElementInternal, changeStart: number, changeRangeOldEnd: number, delta: number): void { - if (element) { + function updateTokenPositionsAndMarkElements(element: ISyntaxElementInternal, changeStart: number, changeRangeOldEnd: number, delta: number, fullStart: number): void { // First, try to skip past any elements that we dont' need to move. We don't need to // move any elements that don't start after the end of the change range. - if (fullStart(element) > changeRangeOldEnd) { - // Note, we only move elements that are truly after the end of the change range. - // We consider elements that are touching the end of the change range to be unusable. - forceUpdateTokenPositionsForElement(element, delta); - } - else { - // Check if the element intersects the change range. If it does, then it is not - // reusable. Also, we'll need to recurse to see what constituent portions we may - // be able to use. - if (fullEnd(element) >= changeStart) { - element.intersectsChange = true; + if (fullStart > changeRangeOldEnd) { + // Note, we only move elements that are truly after the end of the change range. + // We consider elements that are touching the end of the change range to be unusable. + forceUpdateTokenPositionsForElement(element, delta); + } + else { + // Check if the element intersects the change range. If it does, then it is not + // reusable. Also, we'll need to recurse to see what constituent portions we may + // be able to use. + var fullEnd = fullStart + fullWidth(element); + if (fullEnd >= changeStart) { + element.intersectsChange = true; - for (var i = 0, n = childCount(element); i < n; i++) { - updateTokenPositionsAndMarkElements(childAt(element, i), changeStart, changeRangeOldEnd, delta); + for (var i = 0, n = childCount(element); i < n; i++) { + var child = childAt(element, i); + if (child) { + updateTokenPositionsAndMarkElements(child, changeStart, changeRangeOldEnd, delta, fullStart); + fullStart += fullWidth(child); } } - // else { - // This element ended strictly before the edited range. We don't need to do anything - // with it. - // } } + // else { + // This element ended strictly before the edited range. We don't need to do anything + // with it. + // } } } From 7a1d2fbf7b5f56cf14d8273338f8a68756970513 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 21 Nov 2014 14:59:45 -0800 Subject: [PATCH 07/12] Simplify incremental parser. --- src/services/syntax/incrementalParser.ts | 110 ++++++++++------------- 1 file changed, 46 insertions(+), 64 deletions(-) diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index 8af373ce5da..6b562e51c9d 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -172,7 +172,7 @@ module TypeScript.IncrementalParser { return _outstandingRewindPointCount > 0; } - function canReadFromOldSourceUnit() { + function trySynchronizeCursorToPosition() { // If we're currently pinned, then do not want to touch the cursor. Here's why. First, // recall that we're 'pinned' when we're speculatively parsing. So say we were to allow // returning old nodes/tokens while speculatively parsing. Then, the parser might start @@ -194,64 +194,11 @@ module TypeScript.IncrementalParser { return false; } - // If our current absolute position is in the middle of the changed range in the new text - // then we definitely can't read from the old source unit right now. var absolutePos = absolutePosition(); - //if (_changeRangeNewSpan.intersectsWithPosition(absolutePos)) { - // return false; - //} - - // First, try to sync up with the new text if we're behind. - syncCursorToNewTextIfBehind(); - - // Now see what the position is of the node we're considering reusing. If it's at our - // current position then we can *potentialy* reuse it. We'll still need to do more - // viability checks later on. This will happen in tryGetNodeFromOldSourceUnit or - // tryGetTokenFromOldSourceUnit. - var currentNodeOrToken = _oldSourceUnitCursor.currentNodeOrToken(); - - // we should never return a node or token that intersects the change range. - Debug.assert(currentNodeOrToken === undefined || !(currentNodeOrToken).intersectsChange); - return currentNodeOrToken && fullStart(currentNodeOrToken) === absolutePos; - } - - function currentNode(): ISyntaxNode { - if (canReadFromOldSourceUnit()) { - // Try to read a node. If we can't then our caller will call back in and just try - // to get a token. - var node = tryGetNodeFromOldSourceUnit(); - if (node) { - return node; - } - } - - // Either we were ahead of the old text, or we were pinned. No node can be read here. - return undefined; - } - - function currentToken(): ISyntaxToken { - if (canReadFromOldSourceUnit()) { - var token = tryGetTokenFromOldSourceUnit(); - if (token) { - return token; - } - } - - // Either we couldn't read from the old source unit, or we weren't able to successfully - // get a token from it. In this case we need to read a token from the underlying text. - return _scannerParserSource.currentToken(); - } - - function currentContextualToken(): ISyntaxToken { - // Just delegate to the underlying source to handle - return _scannerParserSource.currentContextualToken(); - } - - function syncCursorToNewTextIfBehind() { while (true) { if (_oldSourceUnitCursor.isFinished()) { - // Can't sync up if the cursor is finished. - return; + // Can't synchronize the cursor to the current position if the cursor is finished. + return false; } // Start with the current node or token the cursor is pointing at. @@ -273,15 +220,17 @@ module TypeScript.IncrementalParser { continue; } - var absolutePos = absolutePosition(); - var currentNodeOrTokenFullStart = fullStart(currentNodeOrToken); + if (currentNodeOrTokenFullStart === absolutePos) { + // We were able to synchronize the cursor to the current position. We can + // read from the cursor + return true; + } + if (currentNodeOrTokenFullStart >= absolutePos) { - // The node or token is either where we're at, or ahead of where we're at. If - // it's where we're at, there's no work to be done. If it's ahead of where - // we're at, we need to rescan tokens until we catch up. in either case, - // we don't need to sync the cursor. - return; + // The node or token is ahead of the current position. We'll need to rescan + // tokens until we catch up. + return false; } // The node or is behind the current position we're at in the text. @@ -305,6 +254,39 @@ module TypeScript.IncrementalParser { } } + + function currentNode(): ISyntaxNode { + if (trySynchronizeCursorToPosition()) { + // Try to read a node. If we can't then our caller will call back in and just try + // to get a token. + var node = tryGetNodeFromOldSourceUnit(); + if (node) { + return node; + } + } + + // Either we were ahead of the old text, or we were pinned. No node can be read here. + return undefined; + } + + function currentToken(): ISyntaxToken { + if (trySynchronizeCursorToPosition()) { + var token = tryGetTokenFromOldSourceUnit(); + if (token) { + return token; + } + } + + // Either we couldn't read from the old source unit, or we weren't able to successfully + // get a token from it. In this case we need to read a token from the underlying text. + return _scannerParserSource.currentToken(); + } + + function currentContextualToken(): ISyntaxToken { + // Just delegate to the underlying source to handle + return _scannerParserSource.currentContextualToken(); + } + function tryGetNodeFromOldSourceUnit(): ISyntaxNode { // Debug.assert(canReadFromOldSourceUnit()); @@ -369,7 +351,7 @@ module TypeScript.IncrementalParser { } function peekToken(n: number): ISyntaxToken { - if (canReadFromOldSourceUnit()) { + if (trySynchronizeCursorToPosition()) { var token = tryPeekTokenFromOldSourceUnit(n); if (token) { return token; From eb35cd5d592193182ed15b8e1a3640e8c303ef79 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 21 Nov 2014 15:26:00 -0800 Subject: [PATCH 08/12] Speed up incremental parser. --- src/services/syntax/incrementalParser.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index 6b562e51c9d..83637f37197 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -700,7 +700,7 @@ module TypeScript.IncrementalParser { } var tokenCollectorWalker = new TokenCollectorWalker(); - function updateTokenPositionsAndMarkElements(element: ISyntaxElementInternal, changeStart: number, changeRangeOldEnd: number, delta: number, fullStart: number): void { + function updateTokenPositionsAndMarkElements(element: ISyntaxElement, changeStart: number, changeRangeOldEnd: number, delta: number, fullStart: number): void { // First, try to skip past any elements that we dont' need to move. We don't need to // move any elements that don't start after the end of the change range. if (fullStart > changeRangeOldEnd) { @@ -714,15 +714,26 @@ module TypeScript.IncrementalParser { // be able to use. var fullEnd = fullStart + fullWidth(element); if (fullEnd >= changeStart) { - element.intersectsChange = true; + (element).intersectsChange = true; - for (var i = 0, n = childCount(element); i < n; i++) { - var child = childAt(element, i); - if (child) { + if (isList(element)) { + var list = element; + for (var i = 0, n = list.length; i < n; i++) { + var child: ISyntaxElement = list[i]; updateTokenPositionsAndMarkElements(child, changeStart, changeRangeOldEnd, delta, fullStart); fullStart += fullWidth(child); } } + else if (isNode(element)) { + var node = element; + for (var i = 0, n = node.childCount; i < n; i++) { + var child = node.childAt(i); + if (child) { + updateTokenPositionsAndMarkElements(child, changeStart, changeRangeOldEnd, delta, fullStart); + fullStart += fullWidth(child); + } + } + } } // else { // This element ended strictly before the edited range. We don't need to do anything From ca5a01b3aeb79703ebb691a99d02b8dd2da4fe06 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 21 Nov 2014 15:29:13 -0800 Subject: [PATCH 09/12] Make 'kind' non-enumerable. --- src/services/syntax/syntaxList.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/syntax/syntaxList.ts b/src/services/syntax/syntaxList.ts index 4b11518c239..aa6d8112952 100644 --- a/src/services/syntax/syntaxList.ts +++ b/src/services/syntax/syntaxList.ts @@ -38,7 +38,7 @@ module TypeScript { module TypeScript.Syntax { function addArrayPrototypeValue(name: string, val: any) { if (Object.defineProperty && (Array.prototype)[name] === undefined) { - Object.defineProperty(Array.prototype, name, { value: val, writable: false }); + Object.defineProperty(Array.prototype, name, { value: val, writable: false, enumerable: false }); } else { (Array.prototype)[name] = val; From 3bcec5e3fdb31743e255ee03dabd9967ddbef6a0 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 21 Nov 2014 15:48:06 -0800 Subject: [PATCH 10/12] Remove unused asserts. --- src/services/syntax/incrementalParser.ts | 39 +++--------------------- 1 file changed, 5 insertions(+), 34 deletions(-) diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index 83637f37197..f6288589698 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -104,10 +104,6 @@ module TypeScript.IncrementalParser { // start of the tree. for (var i = 0; start > 0 && i <= maxLookahead; i++) { var token = findToken(sourceUnit, start); - - // Debug.assert(token.kind !== SyntaxKind.None); - // Debug.assert(token.kind() === SyntaxKind.EndOfFileToken || token.fullWidth() > 0); - var position = token.fullStart(); start = Math.max(0, position - 1); @@ -254,7 +250,6 @@ module TypeScript.IncrementalParser { } } - function currentNode(): ISyntaxNode { if (trySynchronizeCursorToPosition()) { // Try to read a node. If we can't then our caller will call back in and just try @@ -288,8 +283,6 @@ module TypeScript.IncrementalParser { } function tryGetNodeFromOldSourceUnit(): ISyntaxNode { - // Debug.assert(canReadFromOldSourceUnit()); - // Keep moving the cursor down to the first node that is safe to return. A node is // safe to return if: // a) it does not contain skipped text. @@ -303,9 +296,7 @@ module TypeScript.IncrementalParser { return undefined; } - var isIncrementallyUnusuable = TypeScript.isIncrementallyUnusable(node); - if (!isIncrementallyUnusuable) { - + if (!TypeScript.isIncrementallyUnusable(node)) { // Didn't contain anything that would make it unusable. Awesome. This is // a node we can reuse. return node; @@ -320,9 +311,10 @@ module TypeScript.IncrementalParser { function canReuseTokenFromOldSourceUnit(token: ISyntaxToken): boolean { // A token is safe to return if: - // a) it does not contain skipped text. - // b) it is not zero width. - // c) it is not a contextual parser token. + // a) it did not intersect the change range. + // b) it does not contain skipped text. + // c) it is not zero width. + // d) it is not a contextual parser token. // // NOTE: It is safe to get a token regardless of what our strict context was/is. That's // because the strict context doesn't change what tokens are scanned, only how the @@ -342,8 +334,6 @@ module TypeScript.IncrementalParser { } function tryGetTokenFromOldSourceUnit(): ISyntaxToken { - // Debug.assert(canReadFromOldSourceUnit()); - // get the current token that the cursor is pointing at. var token = _oldSourceUnitCursor.currentToken(); @@ -363,8 +353,6 @@ module TypeScript.IncrementalParser { } function tryPeekTokenFromOldSourceUnit(n: number): ISyntaxToken { - // Debug.assert(canReadFromOldSourceUnit()); - // clone the existing cursor so we can move it forward and then restore ourselves back // to where we started from. @@ -512,7 +500,6 @@ module TypeScript.IncrementalParser { // Makes this cursor into a deep copy of the cursor passed in. function deepCopyFrom(other: SyntaxCursor): void { - // Debug.assert(currentPieceIndex === -1); for (var i = 0, n = other.pieces.length; i < n; i++) { var piece = other.pieces[i]; @@ -522,8 +509,6 @@ module TypeScript.IncrementalParser { pushElement(piece.element, piece.indexInParent); } - - // Debug.assert(currentPieceIndex === other.currentPieceIndex); } function isFinished(): boolean { @@ -538,9 +523,6 @@ module TypeScript.IncrementalParser { var result = pieces[currentPieceIndex].element; // The current element must always be a node or a token. - // Debug.assert(result !== undefined); - // Debug.assert(result.isNode() || result.isToken()); - return result; } @@ -564,9 +546,6 @@ module TypeScript.IncrementalParser { return; } - // The last element must be a token or a node. - // Debug.assert(isNode(nodeOrToken)); - // Either the node has some existent child, then move to it. if it doesn't, then it's // an empty node. Conceptually the first child of an empty node is really just the // next sibling of the empty node. @@ -585,8 +564,6 @@ module TypeScript.IncrementalParser { // This element must have been an empty node. Moving to its 'first child' is equivalent to just // moving to the next sibling. - - // Debug.assert(fullWidth(nodeOrToken) === 0); moveToNextSibling(); } @@ -630,15 +607,11 @@ module TypeScript.IncrementalParser { if (isList(element)) { // We cannot ever get an empty list in our piece path. Empty lists are 'shared' and // we make sure to filter that out before pushing any children. - // Debug.assert(childCount(element) > 0); - pushElement(childAt(element, 0), /*indexInParent:*/ 0); } } function pushElement(element: ISyntaxElement, indexInParent: number): void { - // Debug.assert(element !== undefined); - // Debug.assert(indexInParent >= 0); currentPieceIndex++; // Reuse an existing piece if we have one. Otherwise, push a new piece to our list. @@ -660,7 +633,6 @@ module TypeScript.IncrementalParser { continue; } - // Debug.assert(isToken(element)); return; } } @@ -669,7 +641,6 @@ module TypeScript.IncrementalParser { moveToFirstToken(); var element = currentNodeOrToken(); - // Debug.assert(element === undefined || element.isToken()); return element; } From b590ceb6b33c48acc7ba32e8c777c87162fd0e33 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 21 Nov 2014 16:35:19 -0800 Subject: [PATCH 11/12] Fixing check. --- src/services/syntax/incrementalParser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index f6288589698..68c492853dd 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -223,7 +223,7 @@ module TypeScript.IncrementalParser { return true; } - if (currentNodeOrTokenFullStart >= absolutePos) { + if (currentNodeOrTokenFullStart > absolutePos) { // The node or token is ahead of the current position. We'll need to rescan // tokens until we catch up. return false; From f10794be742c7fd5b24ab6db06b52f20389f5ee5 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Fri, 21 Nov 2014 16:42:49 -0800 Subject: [PATCH 12/12] Simplify rewind points. --- src/services/syntax/incrementalParser.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/services/syntax/incrementalParser.ts b/src/services/syntax/incrementalParser.ts index 68c492853dd..273b3eec5fe 100644 --- a/src/services/syntax/incrementalParser.ts +++ b/src/services/syntax/incrementalParser.ts @@ -128,12 +128,7 @@ module TypeScript.IncrementalParser { var rewindPoint = _scannerParserSource.getRewindPoint(); // Clone our cursor. That way we can restore to that point if the parser needs to rewind. - var oldSourceUnitCursorClone = cloneSyntaxCursor(_oldSourceUnitCursor); - - // Store where we were when the rewind point was created. - rewindPoint.oldSourceUnitCursor = _oldSourceUnitCursor; - - _oldSourceUnitCursor = oldSourceUnitCursorClone; + rewindPoint.oldSourceUnitCursor = cloneSyntaxCursor(_oldSourceUnitCursor); _outstandingRewindPointCount++; return rewindPoint;