scanner for trivia + accept baselines

src/compiler/scanner.ts

@@ -401,6 +401,9 @@ namespace ts {
               case CharacterCodes.greaterThan:
                   // Starts of conflict marker trivia
                   return true;
+              case CharacterCodes.hash:
+                  // Only if its the beginning can we have #! trivia
+                  return pos === 0;
               default:
                   return ch > CharacterCodes.maxAsciiCharacter;
           }
@@ -461,6 +464,13 @@ namespace ts {
                       }
                       break;
 
+                  case CharacterCodes.hash:
+                      if (isShebangTrivia(text, pos)) {
+                          pos = scanShebangTrivia(text, pos);
+                          continue;
+                      }
+                      break;
+
                   default:
                       if (ch > CharacterCodes.maxAsciiCharacter && (isWhiteSpace(ch) || isLineBreak(ch))) {
                           pos++;
@@ -528,6 +538,20 @@ namespace ts {
         return pos;
     }
 
+    let shebangTriviaRegex = /^#!.*/;
+
+    function isShebangTrivia(text: string, pos: number) {
+        // Shebangs check must only be done at the start of the file
+        Debug.assert(pos === 0);
+        return shebangTriviaRegex.test(text);
+    }
+
+    function scanShebangTrivia(text: string, pos: number) {
+        let shebang = shebangTriviaRegex.exec(text)[0];
+        pos = pos + shebang.length;
+        return pos;
+    }
+
     // Extract comments from the given source text starting at the given position. If trailing is
     // false, whitespace is skipped until the first line break and comments between that location
     // and the next token are returned.If trailing is true, comments occurring between the given
@@ -1087,6 +1111,18 @@ namespace ts {
                     return token = SyntaxKind.EndOfFileToken;
                 }
                 let ch = text.charCodeAt(pos);
+
+                // Special handling for shebang
+                if (ch == CharacterCodes.hash &&  pos === 0 && isShebangTrivia(text, pos)) {
+                    pos = scanShebangTrivia(text ,pos);
+                    if (skipTrivia) {
+                        continue;
+                    }
+                    else {
+                        return token = SyntaxKind.ShebangTrivia;
+                    }
+                }
+
                 switch (ch) {
                     case CharacterCodes.lineFeed:
                     case CharacterCodes.carriageReturn:

src/compiler/types.ts

@@ -17,6 +17,7 @@ namespace ts {
     }
 
     // token > SyntaxKind.Identifer => token is a keyword
+    // Also, If you add a new SyntaxKind be sure to keep the `Markers` section at the bottom in sync
     export const enum SyntaxKind {
         Unknown,
         EndOfFileToken,
@@ -24,6 +25,8 @@ namespace ts {
         MultiLineCommentTrivia,
         NewLineTrivia,
         WhitespaceTrivia,
+        // We detect and preserve #! on the first line
+        ShebangTrivia,
         // We detect and provide better error recovery when we encounter a git merge marker.  This
         // allows us to edit files with git-conflict markers in them in a much more pleasant manner.
         ConflictMarkerTrivia,