Remove restriction that you cannot reuse nodes/tokens during incremental parsing while doing speculatively operations.

Great idea from @yuit

This restriction was in place because the old parser would mutate nodes as it created them. i.e. when
creating a node it would set the parent of its children right then.  During incremental parsing, this
couldl be bad because we might take an old node, set its parent to be something else, and then decide
we didn't even want to go down that speculative path to begin with.  Now the parent of some child node
would be mutated.  So we might have a node that pointed to a child that then pointed to a different
parent.

To get around this, we restricted the incremental so that it would not reuse from the previous tree
if we were speculatively parsing.  However, Yui had a very good idea to just move the parent setting
phase to be a postpast (similar to the new compiler).  By doing this, we never mutate nodes as we
parse, and thus we never end up with inconsistent nodes.  The post pass then ensures that all parents
are correct relative to the new tree.

Conflicts:
	src/services/syntax/SyntaxGenerator.js
	src/services/syntax/SyntaxGenerator.js.map
	src/services/syntax/incrementalParser.ts
	src/services/syntax/scanner.ts
	src/services/syntax/syntaxGenerator.ts
	src/services/syntax/syntaxNodes.concrete.generated.ts

src/services/syntax/SyntaxGenerator.js

@@ -1964,17 +1964,23 @@ function generateConstructorFunction(definition) {
     }
     result += ") {\r\n";
     result += "        if (data) { this.__data = data; }\r\n";
+    result += "        this.parent = undefined";
     if (definition.children.length) {
         result += "        ";
         for (var i = 0; i < definition.children.length; i++) {
+<<<<<<< HEAD
             if (i) {
                 result += ", ";
             }
+=======
+            result += ",\r\n";
+>>>>>>> 691a8a7... Remove restriction that you cannot reuse nodes/tokens during incremental parsing while doing speculatively operations.
             var child = definition.children[i];
             result += "this." + child.name + " = " + getSafeName(child);
         }
         result += ";\r\n";
     }
+<<<<<<< HEAD
     if (definition.children.length > 0) {
         result += "        ";
         for (var i = 0; i < definition.children.length; i++) {
@@ -1991,6 +1997,9 @@ function generateConstructorFunction(definition) {
         }
         result += ";\r\n";
     }
+=======
+    result += ";\r\n";
+>>>>>>> 691a8a7... Remove restriction that you cannot reuse nodes/tokens during incremental parsing while doing speculatively operations.
     result += "    };\r\n";
     result += "    " + definition.name + ".prototype.kind = SyntaxKind." + getNameWithoutSuffix(definition) + ";\r\n";
     result += "    " + definition.name + ".prototype.childCount = " + definition.children.length + ";\r\n";

src/services/syntax/incrementalParser.ts

@@ -30,8 +30,6 @@ module TypeScript.IncrementalParser {
         // The cursor we use to navigate through and retrieve nodes and tokens from the old tree.
         var oldSourceUnit = oldSyntaxTree.sourceUnit();
 
-        var _isSpeculativelyParsing = false;
-
         // Start the cursor pointing at the first element in the source unit (if it exists).
         var _oldSourceUnitCursor = getSyntaxCursor();
         if (oldSourceUnit.moduleElements.length > 0) {
@@ -113,21 +111,13 @@ module TypeScript.IncrementalParser {
 
         function tryParse<T extends ISyntaxNode>(callback: () => T): T {
             // Clone our cursor.  That way we can restore to that point if the parser needs to rewind.
-            var savedOldSourceUnitCursor = cloneSyntaxCursor(_oldSourceUnitCursor);
-            var savedIsSpeculativelyParsing = _isSpeculativelyParsing;
-
-            // Mark that we're speculative parsing.  During speculative parsing we cannot ruse 
-            // nodes from the parse tree.  See the comment in trySynchronizeCursorToPosition for
-            // the reasons why.
-            _isSpeculativelyParsing = true;
+            var savedOldSourceUnitCursor = cloneSyntaxCursor(oldSourceUnitCursor);
 
             // Now defer to our underlying scanner source to actually invoke the callback.  That 
             // way, if the parser decides to rewind, both the scanner source and this incremental
             // source will rewind appropriately.
             var result = _scannerParserSource.tryParse(callback);
 
-            _isSpeculativelyParsing = savedIsSpeculativelyParsing;
-
             if (!result) {
                 // We're rewinding. Reset the cursor to what it was when we got the rewind point.  
                 // Make sure to return our existing cursor to the pool so it can be reused.
@@ -143,27 +133,6 @@ module TypeScript.IncrementalParser {
         }
 
         function trySynchronizeCursorToPosition() {
-            // If we're currently pinned, then do not want to touch the cursor.  Here's why.  First,
-            // recall that we're 'pinned' when we're speculatively parsing.  So say we were to allow
-            // returning old nodes/tokens while speculatively parsing. Then, the parser might start
-            // mutating the nodes and tokens we returned (i.e. by setting their parents).   Then, 
-            // when we rewound, those nodes and tokens would still have those updated parents.  
-            // Parents which we just decided we did *not* want to parse (hence why we rewound).  For
-            // Example, say we have something like:
-            //
-            //          var v = f<a,b,c>e;  // note: this is not generic.
-            //
-            // When incrementally parsing, we will need to speculatively parse to determine if the
-            // above is generic.  This will cause us to reuse the "a, b, c" tokens, and set their 
-            // parent to a new type argument list.  A type argument list we will then throw away once
-            // we decide that it isn't actually generic.  We will have now 'broken' the original tree.
-            //
-            // As such, the rule is simple.  We only return nodes/tokens from teh original tree if
-            // we know the parser will accept and consume them and never rewind back before them.
-            if (_isSpeculativelyParsing) {
-                return false;
-            }
-
             var absolutePos = absolutePosition();
             while (true) {
                 if (_oldSourceUnitCursor.isFinished()) {
@@ -374,7 +343,7 @@ module TypeScript.IncrementalParser {
             peekToken: peekToken,
             consumeNodeOrToken: consumeNodeOrToken,
             tryParse: tryParse,
-            tokenDiagnostics: tokenDiagnostics,
+            diagnostics: diagnostics
         };
     }
 

src/services/syntax/parser.ts

@@ -292,6 +292,7 @@ module TypeScript.Parser {
 
         function parseSyntaxTreeWorker(isDeclaration: boolean): SyntaxTree {
             var sourceUnit = parseSourceUnit();
+            setupParentsForSyntaxNodeOrToken(sourceUnit);
 
             var allDiagnostics = source.tokenDiagnostics().concat(diagnostics);
             allDiagnostics.sort((a: Diagnostic, b: Diagnostic) => a.start() - b.start());
@@ -299,6 +300,34 @@ module TypeScript.Parser {
             return new SyntaxTree(sourceUnit, isDeclaration, allDiagnostics, source.fileName, source.text, source.languageVersion);
         }
 
+        function setupParentsForElement(element: ISyntaxElement, parent: ISyntaxElement) {
+            if (element) {
+                if (element.parent === parent) {
+                    return;
+                }
+                element.parent = parent;
+
+                if (isList(element)) {
+                    setupParentsForList(<ISyntaxNodeOrToken[]>element);
+                }
+                else {
+                    setupParentsForSyntaxNodeOrToken(<ISyntaxNodeOrToken>element);
+                }
+            }
+        }
+
+        function setupParentsForList(list: ISyntaxNodeOrToken[]) {
+            for (var i = 0, n = list.length; i < n; i++) {
+                setupParentsForElement(list[i], list);
+            }
+        }
+
+        function setupParentsForSyntaxNodeOrToken(nodeOrToken: ISyntaxNodeOrToken) {
+            for (var i = 0, n = nodeOrToken.childCount; i < n; i++) {
+                setupParentsForElement(nodeOrToken.childAt(i), nodeOrToken);
+            }
+        }
+
         function tryParse<T extends ISyntaxNode>(callback: () => T): T {
             // See the comments in IParserRewindPoint for the explanation on why we need to store
             // this data, and what it is used for.

src/services/syntax/syntaxGenerator.ts

@@ -1104,14 +1104,20 @@ function generateConstructorFunction(definition: ITypeDefinition) {
     result += ") {\r\n";
 
     result += "        if (data) { this.__data = data; }\r\n";
-
+    result += "        this.parent = undefined";
     if (definition.children.length) {
         result += "        ";
 
         for (var i = 0; i < definition.children.length; i++) {
+<<<<<<< HEAD
             if (i) {
                 result += ", ";
             }
+=======
+            //if (i) {
+                result += ",\r\n";
+            //}
+>>>>>>> 691a8a7... Remove restriction that you cannot reuse nodes/tokens during incremental parsing while doing speculatively operations.
 
             var child = definition.children[i];
             result += "this." + child.name + " = " + getSafeName(child);
@@ -1120,6 +1126,7 @@ function generateConstructorFunction(definition: ITypeDefinition) {
         result += ";\r\n";
     }
 
+<<<<<<< HEAD
     if (definition.children.length > 0) {
         result += "        ";
 
@@ -1139,6 +1146,9 @@ function generateConstructorFunction(definition: ITypeDefinition) {
         }
         result += ";\r\n";
     }
+=======
+    result += ";\r\n";
+>>>>>>> 691a8a7... Remove restriction that you cannot reuse nodes/tokens during incremental parsing while doing speculatively operations.
 
     result += "    };\r\n";
     result += "    " + definition.name + ".prototype.kind = SyntaxKind." + getNameWithoutSuffix(definition) + ";\r\n";

src/services/syntax/syntaxList.ts

@@ -48,21 +48,21 @@ module TypeScript.Syntax {
     addArrayPrototypeValue("kind", SyntaxKind.List);
 
     export function list<T extends ISyntaxNodeOrToken>(nodes: T[]): T[] {
-        if (nodes !== undefined) {
-            for (var i = 0, n = nodes.length; i < n; i++) {
-                nodes[i].parent = nodes;
-            }
-        }
+        //if (nodes !== undefined) {
+        //    for (var i = 0, n = nodes.length; i < n; i++) {
+        //        nodes[i].parent = nodes;
+        //    }
+        //}
 
         return nodes;
     }
 
     export function separatedList<T extends ISyntaxNodeOrToken>(nodesAndTokens: ISyntaxNodeOrToken[]): ISeparatedSyntaxList<T> {
-        if (nodesAndTokens !== undefined) {
-            for (var i = 0, n = nodesAndTokens.length; i < n; i++) {
-                nodesAndTokens[i].parent = nodesAndTokens;
-            }
-        }
+        //if (nodesAndTokens !== undefined) {
+        //    for (var i = 0, n = nodesAndTokens.length; i < n; i++) {
+        //        nodesAndTokens[i].parent = nodesAndTokens;
+        //    }
+        //}
 
         return <ISeparatedSyntaxList<T>>nodesAndTokens;
     }