Merge pull request #1981 from Microsoft/incrementalChecks.ts

Add additional aggressive checks during incremental parsing.
This commit is contained in:
CyrusNajmabadi
2015-02-09 01:40:19 -05:00
11 changed files with 129 additions and 64 deletions

View File

@@ -336,11 +336,16 @@ module ts {
}
function fixupParentReferences(sourceFile: SourceFile) {
// normally parent references are set during binding.
// however here SourceFile data is used only for syntactic features so running the whole binding process is an overhead.
// walk over the nodes and set parent references
// normally parent references are set during binding. However, for clients that only need
// a syntax tree, and no semantic features, then the binding process is an unnecessary
// overhead. This functions allows us to set all the parents, without all the expense of
// binding.
var parent: Node = sourceFile;
function walk(n: Node): void {
forEachChild(sourceFile, visitNode);
return;
function visitNode(n: Node): void {
// walk down setting parents that differ from the parent we think it should be. This
// allows us to quickly bail out of setting parents for subtrees during incremental
// parsing
@@ -349,30 +354,49 @@ module ts {
var saveParent = parent;
parent = n;
forEachChild(n, walk);
forEachChild(n, visitNode);
parent = saveParent;
}
}
forEachChild(sourceFile, walk);
}
function moveElementEntirelyPastChangeRange(element: IncrementalElement, delta: number) {
function shouldCheckNode(node: Node) {
switch (node.kind) {
case SyntaxKind.StringLiteral:
case SyntaxKind.NumericLiteral:
case SyntaxKind.Identifier:
return true;
}
return false;
}
function moveElementEntirelyPastChangeRange(element: IncrementalElement, delta: number, oldText: string, newText: string, aggressiveChecks: boolean) {
if (element.length) {
visitArray(<IncrementalNodeArray>element);
}
else {
visitNode(<IncrementalNode>element);
}
return;
function visitNode(node: IncrementalNode) {
if (aggressiveChecks && shouldCheckNode(node)) {
var text = oldText.substring(node.pos, node.end);
}
// Ditch any existing LS children we may have created. This way we can avoid
// moving them forward.
node._children = undefined;
node.pos += delta;
node.end += delta;
if (aggressiveChecks && shouldCheckNode(node)) {
Debug.assert(text === newText.substring(node.pos, node.end));
}
forEachChild(node, visitNode, visitArray);
checkNodePositions(node, aggressiveChecks);
}
function visitArray(array: IncrementalNodeArray) {
@@ -459,14 +483,35 @@ module ts {
}
}
function updateTokenPositionsAndMarkElements(node: IncrementalNode, changeStart: number, changeRangeOldEnd: number, changeRangeNewEnd: number, delta: number): void {
visitNode(node);
function checkNodePositions(node: Node, aggressiveChecks: boolean) {
if (aggressiveChecks) {
var pos = node.pos;
forEachChild(node, child => {
Debug.assert(child.pos >= pos);
pos = child.end;
});
Debug.assert(pos <= node.end);
}
}
function updateTokenPositionsAndMarkElements(
sourceFile: IncrementalNode,
changeStart: number,
changeRangeOldEnd: number,
changeRangeNewEnd: number,
delta: number,
oldText: string,
newText: string,
aggressiveChecks: boolean): void {
visitNode(sourceFile);
return;
function visitNode(child: IncrementalNode) {
if (child.pos > changeRangeOldEnd) {
// Node is entirely past the change range. We need to move both its pos and
// end, forward or backward appropriately.
moveElementEntirelyPastChangeRange(child, delta);
moveElementEntirelyPastChangeRange(child, delta, oldText, newText, aggressiveChecks);
return;
}
@@ -480,6 +525,8 @@ module ts {
// Adjust the pos or end (or both) of the intersecting element accordingly.
adjustIntersectingElement(child, changeStart, changeRangeOldEnd, changeRangeNewEnd, delta);
forEachChild(child, visitNode, visitArray);
checkNodePositions(child, aggressiveChecks);
return;
}
@@ -490,7 +537,7 @@ module ts {
if (array.pos > changeRangeOldEnd) {
// Array is entirely after the change range. We need to move it, and move any of
// its children.
moveElementEntirelyPastChangeRange(array, delta);
moveElementEntirelyPastChangeRange(array, delta, oldText, newText, aggressiveChecks);
}
else {
// Check if the element intersects the change range. If it does, then it is not
@@ -513,7 +560,6 @@ module ts {
}
}
function extendToAffectedRange(sourceFile: SourceFile, changeRange: TextChangeRange): TextChangeRange {
// Consider the following code:
// void foo() { /; }
@@ -534,6 +580,7 @@ module ts {
// start of the tree.
for (var i = 0; start > 0 && i <= maxLookahead; i++) {
var nearestNode = findNearestNodeStartingBeforeOrAtPosition(sourceFile, start);
Debug.assert(nearestNode.pos <= start);
var position = nearestNode.pos;
start = Math.max(0, position - 1);
@@ -640,6 +687,22 @@ module ts {
}
}
function checkChangeRange(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange, aggressiveChecks: boolean) {
var oldText = sourceFile.text;
if (textChangeRange) {
Debug.assert((oldText.length - textChangeRange.span.length + textChangeRange.newLength) === newText.length);
if (aggressiveChecks || Debug.shouldAssert(AssertionLevel.VeryAggressive)) {
var oldTextPrefix = oldText.substr(0, textChangeRange.span.start);
var newTextPrefix = newText.substr(0, textChangeRange.span.start);
Debug.assert(oldTextPrefix === newTextPrefix);
var oldTextSuffix = oldText.substring(textSpanEnd(textChangeRange.span), oldText.length);
var newTextSuffix = newText.substring(textSpanEnd(textChangeRangeNewSpan(textChangeRange)), newText.length);
Debug.assert(oldTextSuffix === newTextSuffix);
}
}
}
// Produces a new SourceFile for the 'newText' provided. The 'textChangeRange' parameter
// indicates what changed between the 'text' that this SourceFile has and the 'newText'.
@@ -650,7 +713,10 @@ module ts {
// from this SourceFile that are being held onto may change as a result (including
// becoming detached from any SourceFile). It is recommended that this SourceFile not
// be used once 'update' is called on it.
export function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange): SourceFile {
export function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange, aggressiveChecks?: boolean): SourceFile {
aggressiveChecks = aggressiveChecks || Debug.shouldAssert(AssertionLevel.Aggressive);
checkChangeRange(sourceFile, newText, textChangeRange, aggressiveChecks);
if (textChangeRangeIsUnchanged(textChangeRange)) {
// if the text didn't change, then we can just return our current source file as-is.
return sourceFile;
@@ -659,14 +725,22 @@ module ts {
if (sourceFile.statements.length === 0) {
// If we don't have any statements in the current source file, then there's no real
// way to incrementally parse. So just do a full parse instead.
return parseSourceFile(sourceFile.fileName, newText, sourceFile.languageVersion,/*syntaxCursor*/ undefined, /*setNodeParents*/ true)
return parseSourceFile(sourceFile.fileName, newText, sourceFile.languageVersion, /*syntaxCursor*/ undefined, /*setNodeParents*/ true)
}
var oldText = sourceFile.text;
var syntaxCursor = createSyntaxCursor(sourceFile);
// Make the actual change larger so that we know to reparse anything whose lookahead
// might have intersected the change.
var changeRange = extendToAffectedRange(sourceFile, textChangeRange);
checkChangeRange(sourceFile, newText, changeRange, aggressiveChecks);
// Ensure that extending the affected range only moved the start of the change range
// earlier in the file.
Debug.assert(changeRange.span.start <= textChangeRange.span.start);
Debug.assert(textSpanEnd(changeRange.span) === textSpanEnd(textChangeRange.span));
Debug.assert(textSpanEnd(textChangeRangeNewSpan(changeRange)) === textSpanEnd(textChangeRangeNewSpan(textChangeRange)));
// The is the amount the nodes after the edit range need to be adjusted. It can be
// positive (if the edit added characters), negative (if the edit deleted characters)
@@ -674,8 +748,8 @@ module ts {
var delta = textChangeRangeNewSpan(changeRange).length - changeRange.span.length;
// If we added or removed characters during the edit, then we need to go and adjust all
// the nodes after the edit. Those nodes may move forward down (if we inserted chars)
// or they may move backward (if we deleted chars).
// the nodes after the edit. Those nodes may move forward (if we inserted chars) or they
// may move backward (if we deleted chars).
//
// Doing this helps us out in two ways. First, it means that any nodes/tokens we want
// to reuse are already at the appropriate position in the new text. That way when we
@@ -693,7 +767,7 @@ module ts {
// Also, mark any syntax elements that intersect the changed span. We know, up front,
// that we cannot reuse these elements.
updateTokenPositionsAndMarkElements(<IncrementalNode><Node>sourceFile,
changeRange.span.start, textSpanEnd(changeRange.span), textSpanEnd(textChangeRangeNewSpan(changeRange)), delta);
changeRange.span.start, textSpanEnd(changeRange.span), textSpanEnd(textChangeRangeNewSpan(changeRange)), delta, oldText, newText, aggressiveChecks);
// Now that we've set up our internal incremental state just proceed and parse the
// source file in the normal fashion. When possible the parser will retrieve and
@@ -863,7 +937,6 @@ module ts {
var identifiers: Map<string> = {};
var identifierCount = 0;
var nodeCount = 0;
var scanner: Scanner;
var token: SyntaxKind;
var sourceFile = <SourceFile>createNode(SyntaxKind.SourceFile, /*pos*/ 0);
@@ -956,7 +1029,7 @@ module ts {
var parseErrorBeforeNextFinishedNode: boolean = false;
// Create and prime the scanner before parsing the source elements.
scanner = createScanner(languageVersion, /*skipTrivia*/ true, sourceText, scanError);
var scanner = createScanner(languageVersion, /*skipTrivia*/ true, sourceText, scanError);
token = nextToken();
processReferenceComments(sourceFile);
@@ -975,6 +1048,7 @@ module ts {
fixupParentReferences(sourceFile);
}
syntaxCursor = undefined;
return sourceFile;
function setContextFlag(val: Boolean, flag: ParserContextFlags) {

View File

@@ -115,7 +115,7 @@ module Harness.LanguageService {
version: string,
textChangeRange: ts.TextChangeRange
): ts.SourceFile {
var result = ts.updateLanguageServiceSourceFile(document, scriptSnapshot, version, textChangeRange);
var result = ts.updateLanguageServiceSourceFile(document, scriptSnapshot, version, textChangeRange, /*aggressiveChecks:*/ true);
Utils.assertInvariants(result, /*parent:*/ undefined);
return result;
}

View File

@@ -1624,31 +1624,14 @@ module ts {
export var disableIncrementalParsing = false;
export function updateLanguageServiceSourceFile(sourceFile: SourceFile, scriptSnapshot: IScriptSnapshot, version: string, textChangeRange: TextChangeRange): SourceFile {
if (textChangeRange && Debug.shouldAssert(AssertionLevel.Normal)) {
var oldText = sourceFile.scriptSnapshot;
var newText = scriptSnapshot;
Debug.assert((oldText.getLength() - textChangeRange.span.length + textChangeRange.newLength) === newText.getLength());
if (Debug.shouldAssert(AssertionLevel.VeryAggressive)) {
var oldTextPrefix = oldText.getText(0, textChangeRange.span.start);
var newTextPrefix = newText.getText(0, textChangeRange.span.start);
Debug.assert(oldTextPrefix === newTextPrefix);
var oldTextSuffix = oldText.getText(textSpanEnd(textChangeRange.span), oldText.getLength());
var newTextSuffix = newText.getText(textSpanEnd(textChangeRangeNewSpan(textChangeRange)), newText.getLength());
Debug.assert(oldTextSuffix === newTextSuffix);
}
}
export function updateLanguageServiceSourceFile(sourceFile: SourceFile, scriptSnapshot: IScriptSnapshot, version: string, textChangeRange: TextChangeRange, aggressiveChecks?: boolean): SourceFile {
// If we were given a text change range, and our version or open-ness changed, then
// incrementally parse this file.
if (textChangeRange) {
if (version !== sourceFile.version) {
// Once incremental parsing is ready, then just call into this function.
if (!disableIncrementalParsing) {
var newSourceFile = updateSourceFile(sourceFile, scriptSnapshot.getText(0, scriptSnapshot.getLength()), textChangeRange);
var newSourceFile = updateSourceFile(sourceFile, scriptSnapshot.getText(0, scriptSnapshot.getLength()), textChangeRange, aggressiveChecks);
setSourceFileFields(newSourceFile, scriptSnapshot, version);
// after incremental parsing nameTable might not be up-to-date
// drop it so it can be lazily recreated later

View File

@@ -1404,7 +1404,7 @@ declare module "typescript" {
function createNode(kind: SyntaxKind): Node;
function forEachChild<T>(node: Node, cbNode: (node: Node) => T, cbNodeArray?: (nodes: Node[]) => T): T;
function modifierToFlag(token: SyntaxKind): NodeFlags;
function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange): SourceFile;
function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange, aggressiveChecks?: boolean): SourceFile;
function isEvalOrArgumentsIdentifier(node: Node): boolean;
function createSourceFile(fileName: string, sourceText: string, languageVersion: ScriptTarget, setParentNodes?: boolean): SourceFile;
function isLeftHandSideExpression(expr: Expression): boolean;
@@ -1895,7 +1895,7 @@ declare module "typescript" {
}
function createLanguageServiceSourceFile(fileName: string, scriptSnapshot: IScriptSnapshot, scriptTarget: ScriptTarget, version: string, setNodeParents: boolean): SourceFile;
var disableIncrementalParsing: boolean;
function updateLanguageServiceSourceFile(sourceFile: SourceFile, scriptSnapshot: IScriptSnapshot, version: string, textChangeRange: TextChangeRange): SourceFile;
function updateLanguageServiceSourceFile(sourceFile: SourceFile, scriptSnapshot: IScriptSnapshot, version: string, textChangeRange: TextChangeRange, aggressiveChecks?: boolean): SourceFile;
function createDocumentRegistry(): DocumentRegistry;
function preProcessFile(sourceText: string, readImportFiles?: boolean): PreProcessedFileInfo;
function createLanguageService(host: LanguageServiceHost, documentRegistry?: DocumentRegistry): LanguageService;

View File

@@ -4484,13 +4484,14 @@ declare module "typescript" {
members: SymbolTable;
>members : SymbolTable
>SymbolTable : SymbolTable
>members : SymbolTable
>SymbolTable : SymbolTable
properties: Symbol[];
>properties : Symbol[]
>Symbol : Symbol
callSignatures: Signature[];
>callSignatures : Signature[]
@@ -5895,8 +5896,8 @@ declare module "typescript" {
declare module "typescript" {
function createTypeChecker(host: TypeCheckerHost, produceDiagnostics: boolean): TypeChecker;
>createTypeChecker : (host: TypeCheckerHost, produceDiagnostics: boolean) => TypeChecker
>createTypeChecker : (host: TypeCheckerHost, produceDiagnostics: boolean) => TypeChecker
>host : TypeCheckerHost
>TypeCheckerHost : TypeCheckerHost
>produceDiagnostics : boolean
@@ -5904,6 +5905,7 @@ declare module "typescript" {
}
declare module "typescript" {
function createCompilerHost(options: CompilerOptions): CompilerHost;
>createCompilerHost : (options: CompilerOptions) => CompilerHost

View File

@@ -1435,7 +1435,7 @@ declare module "typescript" {
function createNode(kind: SyntaxKind): Node;
function forEachChild<T>(node: Node, cbNode: (node: Node) => T, cbNodeArray?: (nodes: Node[]) => T): T;
function modifierToFlag(token: SyntaxKind): NodeFlags;
function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange): SourceFile;
function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange, aggressiveChecks?: boolean): SourceFile;
function isEvalOrArgumentsIdentifier(node: Node): boolean;
function createSourceFile(fileName: string, sourceText: string, languageVersion: ScriptTarget, setParentNodes?: boolean): SourceFile;
function isLeftHandSideExpression(expr: Expression): boolean;
@@ -1926,7 +1926,7 @@ declare module "typescript" {
}
function createLanguageServiceSourceFile(fileName: string, scriptSnapshot: IScriptSnapshot, scriptTarget: ScriptTarget, version: string, setNodeParents: boolean): SourceFile;
var disableIncrementalParsing: boolean;
function updateLanguageServiceSourceFile(sourceFile: SourceFile, scriptSnapshot: IScriptSnapshot, version: string, textChangeRange: TextChangeRange): SourceFile;
function updateLanguageServiceSourceFile(sourceFile: SourceFile, scriptSnapshot: IScriptSnapshot, version: string, textChangeRange: TextChangeRange, aggressiveChecks?: boolean): SourceFile;
function createDocumentRegistry(): DocumentRegistry;
function preProcessFile(sourceText: string, readImportFiles?: boolean): PreProcessedFileInfo;
function createLanguageService(host: LanguageServiceHost, documentRegistry?: DocumentRegistry): LanguageService;

View File

@@ -4628,13 +4628,14 @@ declare module "typescript" {
members: SymbolTable;
>members : SymbolTable
>SymbolTable : SymbolTable
>members : SymbolTable
>SymbolTable : SymbolTable
properties: Symbol[];
>properties : Symbol[]
>Symbol : Symbol
callSignatures: Signature[];
>callSignatures : Signature[]
@@ -6039,8 +6040,8 @@ declare module "typescript" {
declare module "typescript" {
function createTypeChecker(host: TypeCheckerHost, produceDiagnostics: boolean): TypeChecker;
>createTypeChecker : (host: TypeCheckerHost, produceDiagnostics: boolean) => TypeChecker
>createTypeChecker : (host: TypeCheckerHost, produceDiagnostics: boolean) => TypeChecker
>host : TypeCheckerHost
>TypeCheckerHost : TypeCheckerHost
>produceDiagnostics : boolean
@@ -6048,6 +6049,7 @@ declare module "typescript" {
}
declare module "typescript" {
function createCompilerHost(options: CompilerOptions): CompilerHost;
>createCompilerHost : (options: CompilerOptions) => CompilerHost

View File

@@ -1436,7 +1436,7 @@ declare module "typescript" {
function createNode(kind: SyntaxKind): Node;
function forEachChild<T>(node: Node, cbNode: (node: Node) => T, cbNodeArray?: (nodes: Node[]) => T): T;
function modifierToFlag(token: SyntaxKind): NodeFlags;
function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange): SourceFile;
function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange, aggressiveChecks?: boolean): SourceFile;
function isEvalOrArgumentsIdentifier(node: Node): boolean;
function createSourceFile(fileName: string, sourceText: string, languageVersion: ScriptTarget, setParentNodes?: boolean): SourceFile;
function isLeftHandSideExpression(expr: Expression): boolean;
@@ -1927,7 +1927,7 @@ declare module "typescript" {
}
function createLanguageServiceSourceFile(fileName: string, scriptSnapshot: IScriptSnapshot, scriptTarget: ScriptTarget, version: string, setNodeParents: boolean): SourceFile;
var disableIncrementalParsing: boolean;
function updateLanguageServiceSourceFile(sourceFile: SourceFile, scriptSnapshot: IScriptSnapshot, version: string, textChangeRange: TextChangeRange): SourceFile;
function updateLanguageServiceSourceFile(sourceFile: SourceFile, scriptSnapshot: IScriptSnapshot, version: string, textChangeRange: TextChangeRange, aggressiveChecks?: boolean): SourceFile;
function createDocumentRegistry(): DocumentRegistry;
function preProcessFile(sourceText: string, readImportFiles?: boolean): PreProcessedFileInfo;
function createLanguageService(host: LanguageServiceHost, documentRegistry?: DocumentRegistry): LanguageService;

View File

@@ -4580,13 +4580,14 @@ declare module "typescript" {
members: SymbolTable;
>members : SymbolTable
>SymbolTable : SymbolTable
>members : SymbolTable
>SymbolTable : SymbolTable
properties: Symbol[];
>properties : Symbol[]
>Symbol : Symbol
callSignatures: Signature[];
>callSignatures : Signature[]
@@ -5991,8 +5992,8 @@ declare module "typescript" {
declare module "typescript" {
function createTypeChecker(host: TypeCheckerHost, produceDiagnostics: boolean): TypeChecker;
>createTypeChecker : (host: TypeCheckerHost, produceDiagnostics: boolean) => TypeChecker
>createTypeChecker : (host: TypeCheckerHost, produceDiagnostics: boolean) => TypeChecker
>host : TypeCheckerHost
>TypeCheckerHost : TypeCheckerHost
>produceDiagnostics : boolean
@@ -6000,6 +6001,7 @@ declare module "typescript" {
}
declare module "typescript" {
function createCompilerHost(options: CompilerOptions): CompilerHost;
>createCompilerHost : (options: CompilerOptions) => CompilerHost

View File

@@ -1473,7 +1473,7 @@ declare module "typescript" {
function createNode(kind: SyntaxKind): Node;
function forEachChild<T>(node: Node, cbNode: (node: Node) => T, cbNodeArray?: (nodes: Node[]) => T): T;
function modifierToFlag(token: SyntaxKind): NodeFlags;
function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange): SourceFile;
function updateSourceFile(sourceFile: SourceFile, newText: string, textChangeRange: TextChangeRange, aggressiveChecks?: boolean): SourceFile;
function isEvalOrArgumentsIdentifier(node: Node): boolean;
function createSourceFile(fileName: string, sourceText: string, languageVersion: ScriptTarget, setParentNodes?: boolean): SourceFile;
function isLeftHandSideExpression(expr: Expression): boolean;
@@ -1964,7 +1964,7 @@ declare module "typescript" {
}
function createLanguageServiceSourceFile(fileName: string, scriptSnapshot: IScriptSnapshot, scriptTarget: ScriptTarget, version: string, setNodeParents: boolean): SourceFile;
var disableIncrementalParsing: boolean;
function updateLanguageServiceSourceFile(sourceFile: SourceFile, scriptSnapshot: IScriptSnapshot, version: string, textChangeRange: TextChangeRange): SourceFile;
function updateLanguageServiceSourceFile(sourceFile: SourceFile, scriptSnapshot: IScriptSnapshot, version: string, textChangeRange: TextChangeRange, aggressiveChecks?: boolean): SourceFile;
function createDocumentRegistry(): DocumentRegistry;
function preProcessFile(sourceText: string, readImportFiles?: boolean): PreProcessedFileInfo;
function createLanguageService(host: LanguageServiceHost, documentRegistry?: DocumentRegistry): LanguageService;

View File

@@ -4753,13 +4753,14 @@ declare module "typescript" {
members: SymbolTable;
>members : SymbolTable
>SymbolTable : SymbolTable
>members : SymbolTable
>SymbolTable : SymbolTable
properties: Symbol[];
>properties : Symbol[]
>Symbol : Symbol
callSignatures: Signature[];
>callSignatures : Signature[]
@@ -6164,8 +6165,8 @@ declare module "typescript" {
declare module "typescript" {
function createTypeChecker(host: TypeCheckerHost, produceDiagnostics: boolean): TypeChecker;
>createTypeChecker : (host: TypeCheckerHost, produceDiagnostics: boolean) => TypeChecker
>createTypeChecker : (host: TypeCheckerHost, produceDiagnostics: boolean) => TypeChecker
>host : TypeCheckerHost
>TypeCheckerHost : TypeCheckerHost
>produceDiagnostics : boolean
@@ -6173,6 +6174,7 @@ declare module "typescript" {
}
declare module "typescript" {
function createCompilerHost(options: CompilerOptions): CompilerHost;
>createCompilerHost : (options: CompilerOptions) => CompilerHost