microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2025-12-11 17:41:26 -06:00
Code Issues Packages Projects Releases 212 Wiki Activity

Do not escape actual template literals on emit (#32844)

Browse Source
This commit is contained in:
Ron Buckton 2019-08-13 15:54:35 -07:00 committed by Ryan Cavanaugh
parent 370a596705
commit a74f109f95
10 changed files with 209 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
src/compiler/factory.ts
View File

@ -1329,27 +1329,97 @@ namespace ts {
: node;
}
export function createTemplateHead(text: string) {
const node = <TemplateHead>createSynthesizedNode(SyntaxKind.TemplateHead);
let rawTextScanner: Scanner | undefined;
const invalidValueSentinel: object = {};
function getCookedText(kind: TemplateLiteralToken["kind"], rawText: string) {
if (!rawTextScanner) {
rawTextScanner = createScanner(ScriptTarget.Latest, /*skipTrivia*/ false, LanguageVariant.Standard);
}
switch (kind) {
case SyntaxKind.NoSubstitutionTemplateLiteral:
rawTextScanner.setText("`" + rawText + "`");
break;
case SyntaxKind.TemplateHead:
// tslint:disable-next-line no-invalid-template-strings
rawTextScanner.setText("`" + rawText + "${");
break;
case SyntaxKind.TemplateMiddle:
// tslint:disable-next-line no-invalid-template-strings
rawTextScanner.setText("}" + rawText + "${");
break;
case SyntaxKind.TemplateTail:
rawTextScanner.setText("}" + rawText + "`");
break;
}
let token = rawTextScanner.scan();
if (token === SyntaxKind.CloseBracketToken) {
token = rawTextScanner.reScanTemplateToken();
}
if (rawTextScanner.isUnterminated()) {
rawTextScanner.setText(undefined);
return invalidValueSentinel;
}
let tokenValue: string | undefined;
switch (token) {
case SyntaxKind.NoSubstitutionTemplateLiteral:
case SyntaxKind.TemplateHead:
case SyntaxKind.TemplateMiddle:
case SyntaxKind.TemplateTail:
tokenValue = rawTextScanner.getTokenValue();
break;
}
if (rawTextScanner.scan() !== SyntaxKind.EndOfFileToken) {
rawTextScanner.setText(undefined);
return invalidValueSentinel;
}
rawTextScanner.setText(undefined);
return tokenValue;
}
function createTemplateLiteralLikeNode(kind: TemplateLiteralToken["kind"], text: string, rawText: string | undefined) {
const node = <TemplateLiteralLikeNode>createSynthesizedNode(kind);
node.text = text;
if (rawText === undefined || text === rawText) {
node.rawText = rawText;
}
else {
const cooked = getCookedText(kind, rawText);
if (typeof cooked === "object") {
return Debug.fail("Invalid raw text");
}
Debug.assert(text === cooked, "Expected argument 'text' to be the normalized (i.e. 'cooked') version of argument 'rawText'.");
node.rawText = rawText;
}
return node;
}
export function createTemplateHead(text: string, rawText?: string) {
const node = <TemplateHead>createTemplateLiteralLikeNode(SyntaxKind.TemplateHead, text, rawText);
node.text = text;
return node;
}
export function createTemplateMiddle(text: string) {
const node = <TemplateMiddle>createSynthesizedNode(SyntaxKind.TemplateMiddle);
export function createTemplateMiddle(text: string, rawText?: string) {
const node = <TemplateMiddle>createTemplateLiteralLikeNode(SyntaxKind.TemplateMiddle, text, rawText);
node.text = text;
return node;
}
export function createTemplateTail(text: string) {
const node = <TemplateTail>createSynthesizedNode(SyntaxKind.TemplateTail);
export function createTemplateTail(text: string, rawText?: string) {
const node = <TemplateTail>createTemplateLiteralLikeNode(SyntaxKind.TemplateTail, text, rawText);
node.text = text;
return node;
}
export function createNoSubstitutionTemplateLiteral(text: string) {
const node = <NoSubstitutionTemplateLiteral>createSynthesizedNode(SyntaxKind.NoSubstitutionTemplateLiteral);
node.text = text;
export function createNoSubstitutionTemplateLiteral(text: string, rawText?: string) {
const node = <NoSubstitutionTemplateLiteral>createTemplateLiteralLikeNode(SyntaxKind.NoSubstitutionTemplateLiteral, text, rawText);
return node;
}

14
src/compiler/parser.ts
View File

@ -2300,9 +2300,19 @@ namespace ts {
return <TemplateMiddle | TemplateTail>fragment;
}
function parseLiteralLikeNode(kind: SyntaxKind): LiteralExpression | LiteralLikeNode {
const node = <LiteralExpression>createNode(kind);
function parseLiteralLikeNode(kind: SyntaxKind): LiteralLikeNode {
const node = <LiteralLikeNode>createNode(kind);
node.text = scanner.getTokenValue();
switch (kind) {
case SyntaxKind.NoSubstitutionTemplateLiteral:
case SyntaxKind.TemplateHead:
case SyntaxKind.TemplateMiddle:
case SyntaxKind.TemplateTail:
const isLast = kind === SyntaxKind.NoSubstitutionTemplateLiteral || kind === SyntaxKind.TemplateTail;
const tokenText = scanner.getTokenText();
(<TemplateLiteralLikeNode>node).rawText = tokenText.substring(1, tokenText.length - (scanner.isUnterminated() ? 0 : isLast ? 1 : 2));
break;
}
if (scanner.hasExtendedUnicodeEscape()) {
node.hasExtendedUnicodeEscape = true;

19
src/compiler/transformers/es2015.ts
View File

@ -3993,18 +3993,21 @@ namespace ts {
*
* @param node The ES6 template literal.
*/
function getRawLiteral(node: LiteralLikeNode) {
function getRawLiteral(node: TemplateLiteralLikeNode) {
// Find original source text, since we need to emit the raw strings of the tagged template.
// The raw strings contain the (escaped) strings of what the user wrote.
// Examples: `\n` is converted to "\\n", a template string with a newline to "\n".
let text = getSourceTextOfNodeFromSourceFile(currentSourceFile, node);
let text = node.rawText;
if (text === undefined) {
text = getSourceTextOfNodeFromSourceFile(currentSourceFile, node);
// text contains the original source, it will also contain quotes ("`"), dolar signs and braces ("${" and "}"),
// thus we need to remove those characters.
// First template piece starts with "`", others with "}"
// Last template piece ends with "`", others with "${"
const isLast = node.kind === SyntaxKind.NoSubstitutionTemplateLiteral || node.kind === SyntaxKind.TemplateTail;
text = text.substring(1, text.length - (isLast ? 1 : 2));
// text contains the original source, it will also contain quotes ("`"), dolar signs and braces ("${" and "}"),
// thus we need to remove those characters.
// First template piece starts with "`", others with "}"
// Last template piece ends with "`", others with "${"
const isLast = node.kind === SyntaxKind.NoSubstitutionTemplateLiteral || node.kind === SyntaxKind.TemplateTail;
text = text.substring(1, text.length - (isLast ? 1 : 2));
}
// Newline normalization:
// ES6 Spec 11.8.6.1 - Static Semantics of TV's and TRV's

12
src/compiler/types.ts
View File

@ -1646,6 +1646,10 @@ namespace ts {
hasExtendedUnicodeEscape?: boolean;
}
export interface TemplateLiteralLikeNode extends LiteralLikeNode {
rawText?: string;
}
// The text property of a LiteralExpression stores the interpreted value of the literal in text form. For a StringLiteral,
// or any literal of a template, this means quotes have been removed and escapes have been converted to actual characters.
// For a NumericLiteral, the stored value is the toString() representation of the number. For example 1, 1.00, and 1e0 are all stored as just "1".
@ -1657,7 +1661,7 @@ namespace ts {
kind: SyntaxKind.RegularExpressionLiteral;
}
export interface NoSubstitutionTemplateLiteral extends LiteralExpression {
export interface NoSubstitutionTemplateLiteral extends LiteralExpression, TemplateLiteralLikeNode {
kind: SyntaxKind.NoSubstitutionTemplateLiteral;
}
@ -1696,17 +1700,17 @@ namespace ts {
kind: SyntaxKind.BigIntLiteral;
}
export interface TemplateHead extends LiteralLikeNode {
export interface TemplateHead extends TemplateLiteralLikeNode {
kind: SyntaxKind.TemplateHead;
parent: TemplateExpression;
}
export interface TemplateMiddle extends LiteralLikeNode {
export interface TemplateMiddle extends TemplateLiteralLikeNode {
kind: SyntaxKind.TemplateMiddle;
parent: TemplateSpan;
}
export interface TemplateTail extends LiteralLikeNode {
export interface TemplateTail extends TemplateLiteralLikeNode {
kind: SyntaxKind.TemplateTail;
parent: TemplateSpan;
}

29
src/compiler/utilities.ts
View File

@ -566,8 +566,6 @@ namespace ts {
return emitNode && emitNode.flags || 0;
}
const escapeNoSubstitutionTemplateLiteralText = compose(escapeString, escapeTemplateSubstitution);
const escapeNonAsciiNoSubstitutionTemplateLiteralText = compose(escapeNonAsciiString, escapeTemplateSubstitution);
export function getLiteralText(node: LiteralLikeNode, sourceFile: SourceFile, neverAsciiEscape: boolean | undefined) {
// If we don't need to downlevel and we can reach the original source text using
// the node's parent reference, then simply get the text as it was originally written.
@ -580,9 +578,7 @@ namespace ts {
// If a NoSubstitutionTemplateLiteral appears to have a substitution in it, the original text
// had to include a backslash: `not \${a} substitution`.
const escapeText = neverAsciiEscape || (getEmitFlags(node) & EmitFlags.NoAsciiEscaping) ?
node.kind === SyntaxKind.NoSubstitutionTemplateLiteral ? escapeNoSubstitutionTemplateLiteralText : escapeString :
node.kind === SyntaxKind.NoSubstitutionTemplateLiteral ? escapeNonAsciiNoSubstitutionTemplateLiteralText : escapeNonAsciiString;
const escapeText = neverAsciiEscape || (getEmitFlags(node) & EmitFlags.NoAsciiEscaping) ? escapeString : escapeNonAsciiString;
// If we can't reach the original source text, use the canonical form if it's a number,
// or a (possibly escaped) quoted form of the original text if it's string-like.
@ -595,15 +591,23 @@ namespace ts {
return '"' + escapeText(node.text, CharacterCodes.doubleQuote) + '"';
}
case SyntaxKind.NoSubstitutionTemplateLiteral:
return "`" + escapeText(node.text, CharacterCodes.backtick) + "`";
case SyntaxKind.TemplateHead:
// tslint:disable-next-line no-invalid-template-strings
return "`" + escapeText(node.text, CharacterCodes.backtick) + "${";
case SyntaxKind.TemplateMiddle:
// tslint:disable-next-line no-invalid-template-strings
return "}" + escapeText(node.text, CharacterCodes.backtick) + "${";
case SyntaxKind.TemplateTail:
return "}" + escapeText(node.text, CharacterCodes.backtick) + "`";
const rawText = (<TemplateLiteralLikeNode>node).rawText || escapeTemplateSubstitution(escapeText(node.text, CharacterCodes.backtick));
switch (node.kind) {
case SyntaxKind.NoSubstitutionTemplateLiteral:
return "`" + rawText + "`";
case SyntaxKind.TemplateHead:
// tslint:disable-next-line no-invalid-template-strings
return "`" + rawText + "${";
case SyntaxKind.TemplateMiddle:
// tslint:disable-next-line no-invalid-template-strings
return "}" + rawText + "${";
case SyntaxKind.TemplateTail:
return "}" + rawText + "`";
}
break;
case SyntaxKind.NumericLiteral:
case SyntaxKind.BigIntLiteral:
case SyntaxKind.RegularExpressionLiteral:
@ -3178,7 +3182,8 @@ namespace ts {
// There is no reason for this other than that JSON.stringify does not handle it either.
const doubleQuoteEscapedCharsRegExp = /[\\\"\u0000-\u001f\t\v\f\b\r\n\u2028\u2029\u0085]/g;
const singleQuoteEscapedCharsRegExp = /[\\\'\u0000-\u001f\t\v\f\b\r\n\u2028\u2029\u0085]/g;
const backtickQuoteEscapedCharsRegExp = /[\\\`\u0000-\u001f\t\v\f\b\r\n\u2028\u2029\u0085]/g;
// Template strings should be preserved as much as possible
const backtickQuoteEscapedCharsRegExp = /[\\\`]/g;
const escapedCharsMap = createMapFromTemplate({
"\t": "\\t",
"\v": "\\v",

24
src/harness/evaluator.ts
View File

@ -2,6 +2,7 @@ namespace evaluator {
declare var Symbol: SymbolConstructor;
const sourceFile = vpath.combine(vfs.srcFolder, "source.ts");
const sourceFileJs = vpath.combine(vfs.srcFolder, "source.js");
function compile(sourceText: string, options?: ts.CompilerOptions) {
const fs = vfs.createFromFileSystem(Harness.IO, /*ignoreCase*/ false);
@ -32,9 +33,8 @@ namespace evaluator {
// Add "asyncIterator" if missing
if (!ts.hasProperty(FakeSymbol, "asyncIterator")) Object.defineProperty(FakeSymbol, "asyncIterator", { value: Symbol.for("Symbol.asyncIterator"), configurable: true });
function evaluate(result: compiler.CompilationResult, globals?: Record<string, any>) {
globals = { Symbol: FakeSymbol, ...globals };
export function evaluateTypeScript(sourceText: string, options?: ts.CompilerOptions, globals?: Record<string, any>) {
const result = compile(sourceText, options);
if (ts.some(result.diagnostics)) {
assert.ok(/*value*/ false, "Syntax error in evaluation source text:\n" + ts.formatDiagnostics(result.diagnostics, {
getCanonicalFileName: file => file,
@ -46,6 +46,12 @@ namespace evaluator {
const output = result.getOutput(sourceFile, "js")!;
assert.isDefined(output);
return evaluateJavaScript(output.text, globals, output.file);
}
export function evaluateJavaScript(sourceText: string, globals?: Record<string, any>, sourceFile = sourceFileJs) {
globals = { Symbol: FakeSymbol, ...globals };
const globalNames: string[] = [];
const globalArgs: any[] = [];
for (const name in globals) {
@ -55,15 +61,11 @@ namespace evaluator {
}
}
const evaluateText = `(function (module, exports, require, __dirname, __filename, ${globalNames.join(", ")}) { ${output.text} })`;
// tslint:disable-next-line:no-eval
const evaluateThunk = eval(evaluateText) as (module: any, exports: any, require: (id: string) => any, dirname: string, filename: string, ...globalArgs: any[]) => void;
const evaluateText = `(function (module, exports, require, __dirname, __filename, ${globalNames.join(", ")}) { ${sourceText} })`;
// tslint:disable-next-line:no-eval no-unused-expression
const evaluateThunk = (void 0, eval)(evaluateText) as (module: any, exports: any, require: (id: string) => any, dirname: string, filename: string, ...globalArgs: any[]) => void;
const module: { exports: any; } = { exports: {} };
evaluateThunk.call(globals, module, module.exports, noRequire, vpath.dirname(output.file), output.file, FakeSymbol, ...globalArgs);
evaluateThunk.call(globals, module, module.exports, noRequire, vpath.dirname(sourceFile), sourceFile, FakeSymbol, ...globalArgs);
return module.exports;
}
export function evaluateTypeScript(sourceText: string, options?: ts.CompilerOptions, globals?: Record<string, any>) {
return evaluate(compile(sourceText, options), globals);
}
}

43
src/testRunner/unittests/transform.ts
View File

@ -67,6 +67,24 @@ namespace ts {
});
}
function testBaselineAndEvaluate(testName: string, test: () => string, onEvaluate: (exports: any) => void) {
describe(testName, () => {
let sourceText!: string;
before(() => {
sourceText = test();
});
after(() => {
sourceText = undefined!;
});
it("compare baselines", () => {
Harness.Baseline.runBaseline(`transformApi/transformsCorrectly.${testName}.js`, sourceText);
});
it("evaluate", () => {
onEvaluate(evaluator.evaluateJavaScript(sourceText));
});
});
}
testBaseline("substitution", () => {
return transformSourceFile(`var a = undefined;`, [replaceUndefinedWithVoid0]);
});
@ -440,6 +458,31 @@ namespace Foo {
});
testBaselineAndEvaluate("templateSpans", () => {
return transpileModule("const x = String.raw`\n\nhello`; exports.stringLength = x.trim().length;", {
compilerOptions: {
target: ScriptTarget.ESNext,
newLine: NewLineKind.CarriageReturnLineFeed,
},
transformers: {
before: [transformSourceFile]
}
}).outputText;
function transformSourceFile(context: TransformationContext): Transformer<SourceFile> {
function visitor(node: Node): VisitResult<Node> {
if (isNoSubstitutionTemplateLiteral(node)) {
return createNoSubstitutionTemplateLiteral(node.text, node.rawText);
}
else {
return visitEachChild(node, visitor, context);
}
}
return sourceFile => visitNode(sourceFile, visitor, isSourceFile);
}
}, exports => {
assert.equal(exports.stringLength, 5);
});
});
}

19
tests/baselines/reference/api/tsserverlibrary.d.ts vendored
View File

@ -997,13 +997,16 @@ declare namespace ts {
isUnterminated?: boolean;
hasExtendedUnicodeEscape?: boolean;
}
export interface TemplateLiteralLikeNode extends LiteralLikeNode {
rawText?: string;
}
export interface LiteralExpression extends LiteralLikeNode, PrimaryExpression {
_literalExpressionBrand: any;
}
export interface RegularExpressionLiteral extends LiteralExpression {
kind: SyntaxKind.RegularExpressionLiteral;
}
export interface NoSubstitutionTemplateLiteral extends LiteralExpression {
export interface NoSubstitutionTemplateLiteral extends LiteralExpression, TemplateLiteralLikeNode {
kind: SyntaxKind.NoSubstitutionTemplateLiteral;
}
export enum TokenFlags {
@ -1020,15 +1023,15 @@ declare namespace ts {
export interface BigIntLiteral extends LiteralExpression {
kind: SyntaxKind.BigIntLiteral;
}
export interface TemplateHead extends LiteralLikeNode {
export interface TemplateHead extends TemplateLiteralLikeNode {
kind: SyntaxKind.TemplateHead;
parent: TemplateExpression;
}
export interface TemplateMiddle extends LiteralLikeNode {
export interface TemplateMiddle extends TemplateLiteralLikeNode {
kind: SyntaxKind.TemplateMiddle;
parent: TemplateSpan;
}
export interface TemplateTail extends LiteralLikeNode {
export interface TemplateTail extends TemplateLiteralLikeNode {
kind: SyntaxKind.TemplateTail;
parent: TemplateSpan;
}
@ -3921,10 +3924,10 @@ declare namespace ts {
function updateConditional(node: ConditionalExpression, condition: Expression, questionToken: Token<SyntaxKind.QuestionToken>, whenTrue: Expression, colonToken: Token<SyntaxKind.ColonToken>, whenFalse: Expression): ConditionalExpression;
function createTemplateExpression(head: TemplateHead, templateSpans: ReadonlyArray<TemplateSpan>): TemplateExpression;
function updateTemplateExpression(node: TemplateExpression, head: TemplateHead, templateSpans: ReadonlyArray<TemplateSpan>): TemplateExpression;
function createTemplateHead(text: string): TemplateHead;
function createTemplateMiddle(text: string): TemplateMiddle;
function createTemplateTail(text: string): TemplateTail;
function createNoSubstitutionTemplateLiteral(text: string): NoSubstitutionTemplateLiteral;
function createTemplateHead(text: string, rawText?: string): TemplateHead;
function createTemplateMiddle(text: string, rawText?: string): TemplateMiddle;
function createTemplateTail(text: string, rawText?: string): TemplateTail;
function createNoSubstitutionTemplateLiteral(text: string, rawText?: string): NoSubstitutionTemplateLiteral;
function createYield(expression?: Expression): YieldExpression;
function createYield(asteriskToken: AsteriskToken | undefined, expression: Expression): YieldExpression;
function updateYield(node: YieldExpression, asteriskToken: AsteriskToken | undefined, expression: Expression): YieldExpression;

19
tests/baselines/reference/api/typescript.d.ts vendored
View File

@ -997,13 +997,16 @@ declare namespace ts {
isUnterminated?: boolean;
hasExtendedUnicodeEscape?: boolean;
}
export interface TemplateLiteralLikeNode extends LiteralLikeNode {
rawText?: string;
}
export interface LiteralExpression extends LiteralLikeNode, PrimaryExpression {
_literalExpressionBrand: any;
}
export interface RegularExpressionLiteral extends LiteralExpression {
kind: SyntaxKind.RegularExpressionLiteral;
}
export interface NoSubstitutionTemplateLiteral extends LiteralExpression {
export interface NoSubstitutionTemplateLiteral extends LiteralExpression, TemplateLiteralLikeNode {
kind: SyntaxKind.NoSubstitutionTemplateLiteral;
}
export enum TokenFlags {
@ -1020,15 +1023,15 @@ declare namespace ts {
export interface BigIntLiteral extends LiteralExpression {
kind: SyntaxKind.BigIntLiteral;
}
export interface TemplateHead extends LiteralLikeNode {
export interface TemplateHead extends TemplateLiteralLikeNode {
kind: SyntaxKind.TemplateHead;
parent: TemplateExpression;
}
export interface TemplateMiddle extends LiteralLikeNode {
export interface TemplateMiddle extends TemplateLiteralLikeNode {
kind: SyntaxKind.TemplateMiddle;
parent: TemplateSpan;
}
export interface TemplateTail extends LiteralLikeNode {
export interface TemplateTail extends TemplateLiteralLikeNode {
kind: SyntaxKind.TemplateTail;
parent: TemplateSpan;
}
@ -3921,10 +3924,10 @@ declare namespace ts {
function updateConditional(node: ConditionalExpression, condition: Expression, questionToken: Token<SyntaxKind.QuestionToken>, whenTrue: Expression, colonToken: Token<SyntaxKind.ColonToken>, whenFalse: Expression): ConditionalExpression;
function createTemplateExpression(head: TemplateHead, templateSpans: ReadonlyArray<TemplateSpan>): TemplateExpression;
function updateTemplateExpression(node: TemplateExpression, head: TemplateHead, templateSpans: ReadonlyArray<TemplateSpan>): TemplateExpression;
function createTemplateHead(text: string): TemplateHead;
function createTemplateMiddle(text: string): TemplateMiddle;
function createTemplateTail(text: string): TemplateTail;
function createNoSubstitutionTemplateLiteral(text: string): NoSubstitutionTemplateLiteral;
function createTemplateHead(text: string, rawText?: string): TemplateHead;
function createTemplateMiddle(text: string, rawText?: string): TemplateMiddle;
function createTemplateTail(text: string, rawText?: string): TemplateTail;
function createNoSubstitutionTemplateLiteral(text: string, rawText?: string): NoSubstitutionTemplateLiteral;
function createYield(expression?: Expression): YieldExpression;
function createYield(asteriskToken: AsteriskToken | undefined, expression: Expression): YieldExpression;
function updateYield(node: YieldExpression, asteriskToken: AsteriskToken | undefined, expression: Expression): YieldExpression;

4
tests/baselines/reference/transformApi/transformsCorrectly.templateSpans.js Normal file
View File

@ -0,0 +1,4 @@
const x = String.raw `
hello`;
exports.stringLength = x.trim().length;