Fix unicode escapes in jsx identifiers and extended unicode characters in jsdoc (#32716)

* Fix unicode escapes in jsx identifiers and extended unicode characters in jsdoc

* Support unicode escapes in JSDoc

* Add tests for extended escapes

src/compiler/parser.ts

@@ -7305,10 +7305,14 @@ namespace ts {
                         return createMissingNode<Identifier>(SyntaxKind.Identifier, /*reportAtCurrentPosition*/ !message, message || Diagnostics.Identifier_expected);
                     }
 
+                    identifierCount++;
                     const pos = scanner.getTokenPos();
                     const end = scanner.getTextPos();
                     const result = <Identifier>createNode(SyntaxKind.Identifier, pos);
-                    result.escapedText = escapeLeadingUnderscores(scanner.getTokenText());
+                    if (token() !== SyntaxKind.Identifier) {
+                        result.originalKeywordKind = token();
+                    }
+                    result.escapedText = escapeLeadingUnderscores(internIdentifier(scanner.getTokenValue()));
                     finishNode(result, end);
 
                     nextTokenJSDoc();

src/compiler/scanner.ts

@@ -1015,7 +1015,7 @@ namespace ts {
         }
 
         function checkForIdentifierStartAfterNumericLiteral(numericStart: number, isScientific?: boolean) {
-            if (!isIdentifierStart(text.charCodeAt(pos), languageVersion)) {
+            if (!isIdentifierStart(codePointAt(text, pos), languageVersion)) {
                 return;
             }
 
@@ -2063,17 +2063,22 @@ namespace ts {
         // they allow dashes
         function scanJsxIdentifier(): SyntaxKind {
             if (tokenIsIdentifierOrKeyword(token)) {
-                const firstCharPosition = pos;
+                // An identifier or keyword has already been parsed - check for a `-` and then append it and everything after it to the token
+                // Do note that this means that `scanJsxIdentifier` effectively _mutates_ the visible token without advancing to a new token
+                // Any caller should be expecting this behavior and should only read the pos or token value after calling it.
                 while (pos < end) {
                     const ch = text.charCodeAt(pos);
-                    if (ch === CharacterCodes.minus || ((firstCharPosition === pos) ? isIdentifierStart(ch, languageVersion) : isIdentifierPart(ch, languageVersion))) {
+                    if (ch === CharacterCodes.minus) {
+                        tokenValue += "-";
                         pos++;
+                        continue;
                     }
-                    else {
+                    const oldPos = pos;
+                    tokenValue += scanIdentifierParts(); // reuse `scanIdentifierParts` so unicode escapes are handled
+                    if (pos === oldPos) {
                         break;
                     }
                 }
-                tokenValue += text.substring(firstCharPosition, pos);
             }
             return token;
         }
@@ -2099,8 +2104,8 @@ namespace ts {
                 return token = SyntaxKind.EndOfFileToken;
             }
 
-            const ch = text.charCodeAt(pos);
-            pos++;
+            const ch = codePointAt(text, pos);
+            pos += charSize(ch);
             switch (ch) {
                 case CharacterCodes.tab:
                 case CharacterCodes.verticalTab:
@@ -2138,13 +2143,34 @@ namespace ts {
                     return token = SyntaxKind.DotToken;
                 case CharacterCodes.backtick:
                     return token = SyntaxKind.BacktickToken;
+                case CharacterCodes.backslash:
+                    pos--;
+                    const extendedCookedChar = peekExtendedUnicodeEscape();
+                    if (extendedCookedChar >= 0 && isIdentifierStart(extendedCookedChar, languageVersion)) {
+                        pos += 3;
+                        tokenFlags |= TokenFlags.ExtendedUnicodeEscape;
+                        tokenValue = scanExtendedUnicodeEscape() + scanIdentifierParts();
+                        return token = getIdentifierToken();
+                    }
+
+                    const cookedChar = peekUnicodeEscape();
+                    if (cookedChar >= 0 && isIdentifierStart(cookedChar, languageVersion)) {
+                        pos += 6;
+                        tokenValue = String.fromCharCode(cookedChar) + scanIdentifierParts();
+                        return token = getIdentifierToken();
+                    }
+                    error(Diagnostics.Invalid_character);
+                    pos++;
+                    return token = SyntaxKind.Unknown;
             }
 
-            if (isIdentifierStart(ch, ScriptTarget.Latest)) {
-                while (isIdentifierPart(text.charCodeAt(pos), ScriptTarget.Latest) && pos < end) {
-                    pos++;
-                }
+            if (isIdentifierStart(ch, languageVersion)) {
+                let char = ch;
+                while (pos < end && isIdentifierPart(char = codePointAt(text, pos), languageVersion)) pos += charSize(char);
                 tokenValue = text.substring(tokenPos, pos);
+                if (char === CharacterCodes.backslash) {
+                    tokenValue += scanIdentifierParts();
+                }
                 return token = getIdentifierToken();
             }
             else {
@@ -2265,7 +2291,7 @@ namespace ts {
 
     /* @internal */
     function charSize(ch: number) {
-        if (ch > 0x10000) {
+        if (ch >= 0x10000) {
             return 2;
         }
         return 1;

tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.leadingAsterisk.json

@@ -17,6 +17,7 @@
                 "end": 13,
                 "modifierFlagsCache": 0,
                 "transformFlags": 0,
+                "originalKeywordKind": "TypeKeyword",
                 "escapedText": "type"
             },
             "typeExpression": {

tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.noLeadingAsterisk.json

@@ -17,6 +17,7 @@
                 "end": 13,
                 "modifierFlagsCache": 0,
                 "transformFlags": 0,
+                "originalKeywordKind": "TypeKeyword",
                 "escapedText": "type"
             },
             "typeExpression": {

tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.noReturnType.json

@@ -17,6 +17,7 @@
                 "end": 15,
                 "modifierFlagsCache": 0,
                 "transformFlags": 0,
+                "originalKeywordKind": "ReturnKeyword",
                 "escapedText": "return"
             }
         },

tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.returnTag1.json

@@ -17,6 +17,7 @@
                 "end": 15,
                 "modifierFlagsCache": 0,
                 "transformFlags": 0,
+                "originalKeywordKind": "ReturnKeyword",
                 "escapedText": "return"
             },
             "typeExpression": {

tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.returnTag2.json

@@ -17,6 +17,7 @@
                 "end": 15,
                 "modifierFlagsCache": 0,
                 "transformFlags": 0,
+                "originalKeywordKind": "ReturnKeyword",
                 "escapedText": "return"
             },
             "typeExpression": {

tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.typeTag.json

@@ -17,6 +17,7 @@
                 "end": 13,
                 "modifierFlagsCache": 0,
                 "transformFlags": 0,
+                "originalKeywordKind": "TypeKeyword",
                 "escapedText": "type"
             },
             "typeExpression": {

tests/baselines/reference/extendedUnicodePlaneIdentifiersJSDoc.js

@@ -0,0 +1,19 @@
+//// [file.js]
+/**
+ * Adds
+ * @param {number} 𝑚
+ * @param {number} 𝑀
+ */
+function foo(𝑚, 𝑀) {
+    console.log(𝑀 + 𝑚);
+}
+
+//// [file.js]
+/**
+ * Adds
+ * @param {number} 𝑚
+ * @param {number} 𝑀
+ */
+function foo(𝑚, 𝑀) {
+    console.log(𝑀 + 𝑚);
+}

tests/baselines/reference/extendedUnicodePlaneIdentifiersJSDoc.symbols

@@ -0,0 +1,18 @@
+=== tests/cases/compiler/file.js ===
+/**
+ * Adds
+ * @param {number} 𝑚
+ * @param {number} 𝑀
+ */
+function foo(𝑚, 𝑀) {
+>foo : Symbol(foo, Decl(file.js, 0, 0))
+>𝑚 : Symbol(𝑚, Decl(file.js, 5, 13))
+>𝑀 : Symbol(𝑀, Decl(file.js, 5, 16))
+
+    console.log(𝑀 + 𝑚);
+>console.log : Symbol(Console.log, Decl(lib.dom.d.ts, --, --))
+>console : Symbol(console, Decl(lib.dom.d.ts, --, --))
+>log : Symbol(Console.log, Decl(lib.dom.d.ts, --, --))
+>𝑀 : Symbol(𝑀, Decl(file.js, 5, 16))
+>𝑚 : Symbol(𝑚, Decl(file.js, 5, 13))
+}

tests/baselines/reference/extendedUnicodePlaneIdentifiersJSDoc.types

@@ -0,0 +1,20 @@
+=== tests/cases/compiler/file.js ===
+/**
+ * Adds
+ * @param {number} 𝑚
+ * @param {number} 𝑀
+ */
+function foo(𝑚, 𝑀) {
+>foo : (𝑚: number, 𝑀: number) => void
+>𝑚 : number
+>𝑀 : number
+
+    console.log(𝑀 + 𝑚);
+>console.log(𝑀 + 𝑚) : void
+>console.log : (message?: any, ...optionalParams: any[]) => void
+>console : Console
+>log : (message?: any, ...optionalParams: any[]) => void
+>𝑀 + 𝑚 : number
+>𝑀 : number
+>𝑚 : number
+}

tests/baselines/reference/unicodeEscapesInJSDoc.js

@@ -0,0 +1,33 @@
+//// [file.js]
+/**
+ * @param {number} \u0061
+ * @param {number} a\u0061
+ */
+function foo(a, aa) {
+    console.log(a + aa);
+}
+
+/**
+ * @param {number} \u{0061}
+ * @param {number} a\u{0061}
+ */
+function bar(a, aa) {
+    console.log(a + aa);
+}
+
+
+//// [file.js]
+/**
+ * @param {number} \u0061
+ * @param {number} a\u0061
+ */
+function foo(a, aa) {
+    console.log(a + aa);
+}
+/**
+ * @param {number} \u{0061}
+ * @param {number} a\u{0061}
+ */
+function bar(a, aa) {
+    console.log(a + aa);
+}

tests/baselines/reference/unicodeEscapesInJSDoc.symbols

@@ -0,0 +1,35 @@
+=== tests/cases/compiler/file.js ===
+/**
+ * @param {number} \u0061
+ * @param {number} a\u0061
+ */
+function foo(a, aa) {
+>foo : Symbol(foo, Decl(file.js, 0, 0))
+>a : Symbol(a, Decl(file.js, 4, 13))
+>aa : Symbol(aa, Decl(file.js, 4, 15))
+
+    console.log(a + aa);
+>console.log : Symbol(Console.log, Decl(lib.dom.d.ts, --, --))
+>console : Symbol(console, Decl(lib.dom.d.ts, --, --))
+>log : Symbol(Console.log, Decl(lib.dom.d.ts, --, --))
+>a : Symbol(a, Decl(file.js, 4, 13))
+>aa : Symbol(aa, Decl(file.js, 4, 15))
+}
+
+/**
+ * @param {number} \u{0061}
+ * @param {number} a\u{0061}
+ */
+function bar(a, aa) {
+>bar : Symbol(bar, Decl(file.js, 6, 1))
+>a : Symbol(a, Decl(file.js, 12, 13))
+>aa : Symbol(aa, Decl(file.js, 12, 15))
+
+    console.log(a + aa);
+>console.log : Symbol(Console.log, Decl(lib.dom.d.ts, --, --))
+>console : Symbol(console, Decl(lib.dom.d.ts, --, --))
+>log : Symbol(Console.log, Decl(lib.dom.d.ts, --, --))
+>a : Symbol(a, Decl(file.js, 12, 13))
+>aa : Symbol(aa, Decl(file.js, 12, 15))
+}
+

tests/baselines/reference/unicodeEscapesInJSDoc.types

@@ -0,0 +1,39 @@
+=== tests/cases/compiler/file.js ===
+/**
+ * @param {number} \u0061
+ * @param {number} a\u0061
+ */
+function foo(a, aa) {
+>foo : (a: number, aa: number) => void
+>a : number
+>aa : number
+
+    console.log(a + aa);
+>console.log(a + aa) : void
+>console.log : (message?: any, ...optionalParams: any[]) => void
+>console : Console
+>log : (message?: any, ...optionalParams: any[]) => void
+>a + aa : number
+>a : number
+>aa : number
+}
+
+/**
+ * @param {number} \u{0061}
+ * @param {number} a\u{0061}
+ */
+function bar(a, aa) {
+>bar : (a: number, aa: number) => void
+>a : number
+>aa : number
+
+    console.log(a + aa);
+>console.log(a + aa) : void
+>console.log : (message?: any, ...optionalParams: any[]) => void
+>console : Console
+>log : (message?: any, ...optionalParams: any[]) => void
+>a + aa : number
+>a : number
+>aa : number
+}
+

tests/baselines/reference/unicodeEscapesInJsxtags.js

@@ -0,0 +1,34 @@
+//// [file.tsx]
+import * as React from "react";
+declare global {
+    namespace JSX {
+        interface IntrinsicElements {
+            "a-b": any;
+            "a-c": any;
+        }
+    }
+}
+const Compa = (x: {x: number}) => <div>{"" + x}</div>;
+
+let a = <\u0061></a>; // works
+let ab = <\u0061-b></a-b>; // works
+let ac = <a-\u0063></a-c>; // works
+let compa = <Comp\u0061 x={12} />; // works
+
+let a2 = <\u{0061}></a>; // works
+let ab2 = <\u{0061}-b></a-b>; // works
+let ac2 = <a-\u{0063}></a-c>; // works
+let compa2 = <Comp\u{0061} x={12} />; // works
+
+
+//// [file.js]
+import * as React from "react";
+const Compa = (x) => React.createElement("div", null, "" + x);
+let a = React.createElement("a", null); // works
+let ab = React.createElement("a-b", null); // works
+let ac = React.createElement("a-c", null); // works
+let compa = React.createElement(Comp\u0061, { x: 12 }); // works
+let a2 = React.createElement("a", null); // works
+let ab2 = React.createElement("a-b", null); // works
+let ac2 = React.createElement("a-c", null); // works
+let compa2 = React.createElement(Comp\u{0061}, { x: 12 }); // works

tests/baselines/reference/unicodeEscapesInJsxtags.symbols

@@ -0,0 +1,69 @@
+=== tests/cases/conformance/jsx/file.tsx ===
+import * as React from "react";
+>React : Symbol(React, Decl(file.tsx, 0, 6))
+
+declare global {
+>global : Symbol(global, Decl(file.tsx, 0, 31))
+
+    namespace JSX {
+>JSX : Symbol(JSX, Decl(react.d.ts, 2353, 1), Decl(file.tsx, 1, 16))
+
+        interface IntrinsicElements {
+>IntrinsicElements : Symbol(IntrinsicElements, Decl(react.d.ts, 2368, 78), Decl(file.tsx, 2, 19))
+
+            "a-b": any;
+>"a-b" : Symbol(IntrinsicElements["a-b"], Decl(file.tsx, 3, 37))
+
+            "a-c": any;
+>"a-c" : Symbol(IntrinsicElements["a-c"], Decl(file.tsx, 4, 23))
+        }
+    }
+}
+const Compa = (x: {x: number}) => <div>{"" + x}</div>;
+>Compa : Symbol(Compa, Decl(file.tsx, 9, 5))
+>x : Symbol(x, Decl(file.tsx, 9, 15))
+>x : Symbol(x, Decl(file.tsx, 9, 19))
+>div : Symbol(JSX.IntrinsicElements.div, Decl(react.d.ts, 2400, 45))
+>x : Symbol(x, Decl(file.tsx, 9, 15))
+>div : Symbol(JSX.IntrinsicElements.div, Decl(react.d.ts, 2400, 45))
+
+let a = <\u0061></a>; // works
+>a : Symbol(a, Decl(file.tsx, 11, 3))
+>\u0061 : Symbol(JSX.IntrinsicElements.a, Decl(react.d.ts, 2370, 33))
+>a : Symbol(JSX.IntrinsicElements.a, Decl(react.d.ts, 2370, 33))
+
+let ab = <\u0061-b></a-b>; // works
+>ab : Symbol(ab, Decl(file.tsx, 12, 3))
+>\u0061-b : Symbol(JSX.IntrinsicElements["a-b"], Decl(file.tsx, 3, 37))
+>a-b : Symbol(JSX.IntrinsicElements["a-b"], Decl(file.tsx, 3, 37))
+
+let ac = <a-\u0063></a-c>; // works
+>ac : Symbol(ac, Decl(file.tsx, 13, 3))
+>a-\u0063 : Symbol(JSX.IntrinsicElements["a-c"], Decl(file.tsx, 4, 23))
+>a-c : Symbol(JSX.IntrinsicElements["a-c"], Decl(file.tsx, 4, 23))
+
+let compa = <Comp\u0061 x={12} />; // works
+>compa : Symbol(compa, Decl(file.tsx, 14, 3))
+>Comp\u0061 : Symbol(Compa, Decl(file.tsx, 9, 5))
+>x : Symbol(x, Decl(file.tsx, 14, 23))
+
+let a2 = <\u{0061}></a>; // works
+>a2 : Symbol(a2, Decl(file.tsx, 16, 3))
+>\u{0061} : Symbol(JSX.IntrinsicElements.a, Decl(react.d.ts, 2370, 33))
+>a : Symbol(JSX.IntrinsicElements.a, Decl(react.d.ts, 2370, 33))
+
+let ab2 = <\u{0061}-b></a-b>; // works
+>ab2 : Symbol(ab2, Decl(file.tsx, 17, 3))
+>\u{0061}-b : Symbol(JSX.IntrinsicElements["a-b"], Decl(file.tsx, 3, 37))
+>a-b : Symbol(JSX.IntrinsicElements["a-b"], Decl(file.tsx, 3, 37))
+
+let ac2 = <a-\u{0063}></a-c>; // works
+>ac2 : Symbol(ac2, Decl(file.tsx, 18, 3))
+>a-\u{0063} : Symbol(JSX.IntrinsicElements["a-c"], Decl(file.tsx, 4, 23))
+>a-c : Symbol(JSX.IntrinsicElements["a-c"], Decl(file.tsx, 4, 23))
+
+let compa2 = <Comp\u{0061} x={12} />; // works
+>compa2 : Symbol(compa2, Decl(file.tsx, 19, 3))
+>Comp\u{0061} : Symbol(Compa, Decl(file.tsx, 9, 5))
+>x : Symbol(x, Decl(file.tsx, 19, 26))
+

tests/baselines/reference/unicodeEscapesInJsxtags.types

@@ -0,0 +1,79 @@
+=== tests/cases/conformance/jsx/file.tsx ===
+import * as React from "react";
+>React : typeof React
+
+declare global {
+>global : any
+
+    namespace JSX {
+        interface IntrinsicElements {
+            "a-b": any;
+>"a-b" : any
+
+            "a-c": any;
+>"a-c" : any
+        }
+    }
+}
+const Compa = (x: {x: number}) => <div>{"" + x}</div>;
+>Compa : (x: { x: number; }) => JSX.Element
+>(x: {x: number}) => <div>{"" + x}</div> : (x: { x: number; }) => JSX.Element
+>x : { x: number; }
+>x : number
+><div>{"" + x}</div> : JSX.Element
+>div : any
+>"" + x : string
+>"" : ""
+>x : { x: number; }
+>div : any
+
+let a = <\u0061></a>; // works
+>a : JSX.Element
+><\u0061></a> : JSX.Element
+>\u0061 : JSX.Element
+>a : JSX.Element
+
+let ab = <\u0061-b></a-b>; // works
+>ab : JSX.Element
+><\u0061-b></a-b> : JSX.Element
+>\u0061-b : any
+>a-b : any
+
+let ac = <a-\u0063></a-c>; // works
+>ac : JSX.Element
+><a-\u0063></a-c> : JSX.Element
+>a-\u0063 : any
+>a-c : any
+
+let compa = <Comp\u0061 x={12} />; // works
+>compa : JSX.Element
+><Comp\u0061 x={12} /> : JSX.Element
+>Comp\u0061 : (x: { x: number; }) => JSX.Element
+>x : number
+>12 : 12
+
+let a2 = <\u{0061}></a>; // works
+>a2 : JSX.Element
+><\u{0061}></a> : JSX.Element
+>\u{0061} : JSX.Element
+>a : JSX.Element
+
+let ab2 = <\u{0061}-b></a-b>; // works
+>ab2 : JSX.Element
+><\u{0061}-b></a-b> : JSX.Element
+>\u{0061}-b : any
+>a-b : any
+
+let ac2 = <a-\u{0063}></a-c>; // works
+>ac2 : JSX.Element
+><a-\u{0063}></a-c> : JSX.Element
+>a-\u{0063} : any
+>a-c : any
+
+let compa2 = <Comp\u{0061} x={12} />; // works
+>compa2 : JSX.Element
+><Comp\u{0061} x={12} /> : JSX.Element
+>Comp\u{0061} : (x: { x: number; }) => JSX.Element
+>x : number
+>12 : 12
+

tests/cases/compiler/extendedUnicodePlaneIdentifiersJSDoc.ts

@@ -0,0 +1,13 @@
+// @allowJs: true
+// @checkJs: true
+// @outDir: ./out
+// @target: es2018
+// @filename: file.js
+/**
+ * Adds
+ * @param {number} 𝑚
+ * @param {number} 𝑀
+ */
+function foo(𝑚, 𝑀) {
+    console.log(𝑀 + 𝑚);
+}

tests/cases/compiler/unicodeEscapesInJSDoc.ts

@@ -0,0 +1,20 @@
+// @allowJs: true
+// @checkJs: true
+// @outDir: ./out
+// @target: es2018
+// @filename: file.js
+/**
+ * @param {number} \u0061
+ * @param {number} a\u0061
+ */
+function foo(a, aa) {
+    console.log(a + aa);
+}
+
+/**
+ * @param {number} \u{0061}
+ * @param {number} a\u{0061}
+ */
+function bar(a, aa) {
+    console.log(a + aa);
+}

tests/cases/conformance/jsx/unicodeEscapesInJsxtags.tsx

@@ -0,0 +1,27 @@
+// @filename: file.tsx
+// @jsx: react
+// @noLib: true
+// @skipLibCheck: true
+// @target: es2015
+// @moduleResolution: node
+// @libFiles: react.d.ts,lib.d.ts
+import * as React from "react";
+declare global {
+    namespace JSX {
+        interface IntrinsicElements {
+            "a-b": any;
+            "a-c": any;
+        }
+    }
+}
+const Compa = (x: {x: number}) => <div>{"" + x}</div>;
+
+let a = <\u0061></a>; // works
+let ab = <\u0061-b></a-b>; // works
+let ac = <a-\u0063></a-c>; // works
+let compa = <Comp\u0061 x={12} />; // works
+
+let a2 = <\u{0061}></a>; // works
+let ab2 = <\u{0061}-b></a-b>; // works
+let ac2 = <a-\u{0063}></a-c>; // works
+let compa2 = <Comp\u{0061} x={12} />; // works