diff --git a/Jakefile b/Jakefile index 16330305b6a..039c085ba12 100644 --- a/Jakefile +++ b/Jakefile @@ -84,6 +84,7 @@ var harnessSources = [ ].map(function (f) { return path.join(harnessDirectory, f); }).concat([ + "incrementalParser.ts", "services/colorization.ts", "services/documentRegistry.ts", "services/preProcessFile.ts" diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts index 4f4b7bd96e0..a05b88cda87 100644 --- a/src/compiler/parser.ts +++ b/src/compiler/parser.ts @@ -359,6 +359,7 @@ module ts { case SyntaxKind.Block: case SyntaxKind.TryBlock: case SyntaxKind.FinallyBlock: + return children((node).statements); case SyntaxKind.ModuleBlock: return children((node).statements); case SyntaxKind.SourceFile: diff --git a/src/harness/harness.ts b/src/harness/harness.ts index 4afbc98c48a..f51e492d6b5 100644 --- a/src/harness/harness.ts +++ b/src/harness/harness.ts @@ -111,6 +111,85 @@ module Utils { } }); } + + export function checkInvariants(node: ts.Node, parent: ts.Node): void { + if(node) { + if (node.pos < 0) { + throw new Error("node.pos < 0"); + } + if (node.end < 0) { + throw new Error("node.end < 0"); + } + if (node.end < node.pos) { + throw new Error("node.end < node.pos"); + } + if (node.parent !== parent) { + throw new Error("node.parent !== parent"); + } + if (parent) { + // Make sure each child is contained within the parent. + if (node.pos < parent.pos) { + throw new Error("node.pos < parent.pos"); + } + if (node.end > parent.end) { + throw new Error("node.end > parent.end"); + } + } + + ts.forEachChild(node, child => { + checkInvariants(child, node); + }); + + // Make sure each of the children is in order. + var currentPos = 0; + ts.forEachChild(node, + child => { + if (child.pos < currentPos) { + throw new Error("child.pos < currentPos"); + } + currentPos = child.end; + }, + (array: ts.NodeArray) => { + if (array.pos < node.pos) { + throw new Error("array.pos < node.pos"); + } + if (array.end > node.end) { + throw new Error("array.end > node.end"); + } + + if (array.pos < currentPos) { + throw new Error("array.pos < currentPos"); + } + for (var i = 0, n = array.length; i < n; i++) { + if (array[i].pos < currentPos) { + throw new Error("array[i].pos < currentPos"); + } + currentPos = array[i].end + } + + currentPos = array.end; + }); + + var childNodesAndArrays: any[] = []; + ts.forEachChild(node, child => { childNodesAndArrays.push(child) }, array => { childNodesAndArrays.push(array) }); + + for (var childName in node) { + if (childName === "parent" || childName === "nextContainer" || childName === "modifiers" || childName === "externalModuleIndicator") { + continue; + } + var child = (node)[childName]; + if (isNodeOrArray(child)) { + if (childNodesAndArrays.indexOf(child) < 0) { + throw new Error("Child when forEach'ing over node. " + (ts).SyntaxKind[node.kind] + "-" + childName); + } + } + } + } + } + + function isNodeOrArray(a: any): boolean { + return a !== undefined && typeof a.pos === "number"; + } } module Harness.Path { diff --git a/src/harness/test262Runner.ts b/src/harness/test262Runner.ts index a29c6ac8619..f9c083a2542 100644 --- a/src/harness/test262Runner.ts +++ b/src/harness/test262Runner.ts @@ -21,81 +21,6 @@ class Test262BaselineRunner extends RunnerBase { return Test262BaselineRunner.basePath + "/" + filename; } - private static checkInvariants(node: ts.Node, parent: ts.Node): void { - if (node) { - if (node.pos < 0) { - throw new Error("node.pos < 0"); - } - if (node.end < 0) { - throw new Error("node.end < 0"); - } - if (node.end < node.pos) { - throw new Error("node.end < node.pos"); - } - if (node.parent !== parent) { - throw new Error("node.parent !== parent"); - } - if (parent) { - // Make sure each child is contained within the parent. - if (node.pos < parent.pos) { - throw new Error("node.pos < parent.pos"); - } - if (node.end > parent.end) { - throw new Error("node.end > parent.end"); - } - } - - ts.forEachChild(node, child => { - Test262BaselineRunner.checkInvariants(child, node); - }); - - // Make sure each of the children is in order. - var currentPos = 0; - ts.forEachChild(node, - child => { - if (child.pos < currentPos) { - throw new Error("child.pos < currentPos"); - } - currentPos = child.end; - }, - (array: ts.NodeArray) => { - if (array.pos < node.pos) { - throw new Error("array.pos < node.pos"); - } - if (array.end > node.end) { - throw new Error("array.end > node.end"); - } - - if (array.pos < currentPos) { - throw new Error("array.pos < currentPos"); - } - for (var i = 0, n = array.length; i < n; i++) { - if (array[i].pos < currentPos) { - throw new Error("array[i].pos < currentPos"); - } - currentPos = array[i].end - } - - currentPos = array.end; - }); - - var childNodesAndArrays: any[] = []; - ts.forEachChild(node, child => { childNodesAndArrays.push(child) }, array => { childNodesAndArrays.push(array) }); - - for (var childName in node) { - if (childName === "parent" || childName === "nextContainer" || childName === "modifiers" || childName === "externalModuleIndicator") { - continue; - } - var child = (node)[childName]; - if (Test262BaselineRunner.isNodeOrArray(child)) { - if (childNodesAndArrays.indexOf(child) < 0) { - throw new Error("Child when forEach'ing over node. " + (ts).SyntaxKind[node.kind] + "-" + childName); - } - } - } - } - } - private static serializeSourceFile(file: ts.SourceFile): string { function getKindName(k: number): string { return (ts).SyntaxKind[k] @@ -264,7 +189,7 @@ class Test262BaselineRunner extends RunnerBase { it('satisfies invariants', () => { var sourceFile = testState.checker.getProgram().getSourceFile(Test262BaselineRunner.getTestFilePath(testState.filename)); - Test262BaselineRunner.checkInvariants(sourceFile, /*parent:*/ undefined); + Utils.checkInvariants(sourceFile, /*parent:*/ undefined); }); it('has the expected AST',() => { diff --git a/src/services/services.ts b/src/services/services.ts index b435c554420..07eafbcbeaf 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -1679,7 +1679,7 @@ module ts { var scriptSnapshot = this.hostCache.getScriptSnapshot(filename); var start = new Date().getTime(); - sourceFile = createSourceFileFromScriptSnapshot(filename, scriptSnapshot, getDefaultCompilerOptions(), version, /*isOpen*/ true); + sourceFile = createLanguageServiceSourceFile(filename, scriptSnapshot, getDefaultCompilerOptions(), version, /*isOpen*/ true); this.host.log("SyntaxTreeCache.Initialize: createSourceFile: " + (new Date().getTime() - start)); var start = new Date().getTime(); @@ -1692,7 +1692,7 @@ module ts { var start = new Date().getTime(); sourceFile = !editRange - ? createSourceFileFromScriptSnapshot(filename, scriptSnapshot, getDefaultCompilerOptions(), version, /*isOpen*/ true) + ? createLanguageServiceSourceFile(filename, scriptSnapshot, getDefaultCompilerOptions(), version, /*isOpen*/ true) : this.currentSourceFile.update(scriptSnapshot, version, /*isOpen*/ true, editRange); this.host.log("SyntaxTreeCache.Initialize: updateSourceFile: " + (new Date().getTime() - start)); @@ -1718,7 +1718,7 @@ module ts { } } - function createSourceFileFromScriptSnapshot(filename: string, scriptSnapshot: IScriptSnapshot, settings: CompilerOptions, version: string, isOpen: boolean) { + export function createLanguageServiceSourceFile(filename: string, scriptSnapshot: IScriptSnapshot, settings: CompilerOptions, version: string, isOpen: boolean): SourceFile { return SourceFileObject.createSourceFileObject(filename, scriptSnapshot, settings.target, version, isOpen); } @@ -1769,7 +1769,7 @@ module ts { var bucket = getBucketForCompilationSettings(compilationSettings, /*createIfMissing*/ true); var entry = lookUp(bucket, filename); if (!entry) { - var sourceFile = createSourceFileFromScriptSnapshot(filename, scriptSnapshot, compilationSettings, version, isOpen); + var sourceFile = createLanguageServiceSourceFile(filename, scriptSnapshot, compilationSettings, version, isOpen); bucket[filename] = entry = { sourceFile: sourceFile, diff --git a/tests/cases/unittests/incrementalParser.ts b/tests/cases/unittests/incrementalParser.ts new file mode 100644 index 00000000000..538996bc667 --- /dev/null +++ b/tests/cases/unittests/incrementalParser.ts @@ -0,0 +1,166 @@ +/// +/// + +module ts { + function withChange(text: IScriptSnapshot, start: number, length: number, newText: string): { text: IScriptSnapshot; textChangeRange: TextChangeRange; } { + var contents = text.getText(0, text.getLength()); + var newContents = contents.substr(0, start) + newText + contents.substring(start + length); + + return { text: ScriptSnapshot.fromString(newContents), textChangeRange: new TextChangeRange(new TextSpan(start, length), newText.length) } + } + + function withInsert(text: IScriptSnapshot, start: number, newText: string): { text: IScriptSnapshot; textChangeRange: TextChangeRange; } { + return withChange(text, start, 0, newText); + } + + function withDelete(text: IScriptSnapshot, start: number, length: number): { text: IScriptSnapshot; textChangeRange: TextChangeRange; } { + return withChange(text, start, length, ""); + } + + // NOTE: 'reusedElements' is the expected count of elements reused from the old tree to the new + // tree. It may change as we tweak the parser. If the count increases then that should always + // be a good thing. If it decreases, that's not great (less reusability), but that may be + // unavoidable. If it does decrease an investigation should be done to make sure that things + // are still ok and we're still appropriately reusing most of the tree. + function compareTrees(oldText: IScriptSnapshot, newText: IScriptSnapshot, textChangeRange: TextChangeRange, expectedReusedElements: number = -1): void { + // Create a tree for the new text, in a non-incremental fashion. + var options: CompilerOptions = {}; + options.target = ScriptTarget.ES5; + + var newTree = createLanguageServiceSourceFile(/*fileName:*/ "", newText, options, /*version:*/ "0", /*isOpen:*/ true); + Utils.checkInvariants(newTree, /*parent:*/ undefined); + + // Create a tree for the new text, in an incremental fashion. + var oldTree = createLanguageServiceSourceFile(/*fileName:*/ "", oldText, options, /*version:*/ "0", /*isOpen:*/ true); + Utils.checkInvariants(oldTree, /*parent:*/ undefined); + + var incrementalNewTree = oldTree.update(newText, "1", /*isOpen:*/ true, textChangeRange); + Utils.checkInvariants(incrementalNewTree, /*parent:*/ undefined); + + // We should get the same tree when doign a full or incremental parse. + assertStructuralEquals(newTree, incrementalNewTree); + + // There should be no reused nodes between two trees that are fully parsed. + Debug.assert(reusedElements(oldTree, newTree) === 0); + + if (expectedReusedElements !== -1) { + var actualReusedCount = reusedElements(oldTree, incrementalNewTree); + Debug.assert(actualReusedCount === expectedReusedElements, actualReusedCount + " !== " + expectedReusedElements); + } + } + + function assertStructuralEquals(node1: Node, node2: Node) { + if (node1 === node2) { + return; + } + + if (!node1 || !node2) { + throw new Error("!node1 || !node2"); + } + + if (node1.pos !== node2.pos) { + throw new Error("node1.pos !== node2.pos"); + } + + if (node1.end !== node2.end) { + throw new Error("node1.end !== node2.end"); + } + + if (node1.kind !== node2.kind) { + throw new Error("node1.kind !== node2.kind"); + } + + if (node1.flags !== node2.flags) { + throw new Error("node1.flags !== node2.flags"); + } + + if (node1.parserContextFlags !== node2.parserContextFlags) { + throw new Error("node1.parserContextFlags !== node2.parserContextFlags"); + } + + forEachChild(node1, + child1 => { + var childName = findChildName(node1, child1); + var child2: Node = (node2)[childName]; + + assertStructuralEquals(child1, child2); + }, + (array1: NodeArray) => { + var childName = findChildName(node1, array1); + var array2: NodeArray = (node2)[childName]; + + assertArrayStructuralEquals(array1, array2); + }); + } + + function assertArrayStructuralEquals(array1: NodeArray, array2: NodeArray) { + if (array1 === array2) { + return; + } + + if (!array1 || !array2) { + throw new Error("!array1 || !array2"); + } + + if (array1.pos !== array2.pos) { + throw new Error("array1.pos !== array2.pos"); + } + + if (array1.end !== array2.end) { + throw new Error("array1.end !== array2.end"); + } + + if (array1.length !== array2.length) { + throw new Error("array1.length !== array2.length"); + } + + for (var i = 0, n = array1.length; i < n; i++) { + assertStructuralEquals(array1[i], array2[i]); + } + } + + function findChildName(parent: any, child: any) { + for (var name in parent) { + if (parent.hasOwnProperty(name) && parent[name] === child) { + return name; + } + } + + throw new Error("Could not find child in parent"); + } + + function reusedElements(oldNode: SourceFile, newNode: SourceFile): number { + var allOldElements = collectElements(oldNode); + var allNewElements = collectElements(newNode); + + return filter(allOldElements, v => contains(allNewElements, v)).length; + } + + function collectElements(node: Node) { + var result: Node[] = []; + visit(node); + return result; + + function visit(node: Node) { + result.push(node); + forEachChild(node, visit); + } + } + + describe('Incremental',() => { + it('Inserting into method',() => { + var source = "class C {\r\n" + + " public foo1() { }\r\n" + + " public foo2() {\r\n" + + " return 1;\r\n" + + " }\r\n" + + " public foo3() { }\r\n" + + "}"; + + var oldText = ScriptSnapshot.fromString(source); + var semicolonIndex = source.indexOf(";"); + var newTextAndChange = withInsert(oldText, semicolonIndex, " + 1"); + compareTrees(oldText, newTextAndChange.text, newTextAndChange.textChangeRange, 0); + }); + }); +} \ No newline at end of file