diff --git a/src/compiler/factory.ts b/src/compiler/factory.ts index 8678ba4c122..d0d617b3495 100644 --- a/src/compiler/factory.ts +++ b/src/compiler/factory.ts @@ -1329,27 +1329,97 @@ namespace ts { : node; } - export function createTemplateHead(text: string) { - const node = createSynthesizedNode(SyntaxKind.TemplateHead); + let rawTextScanner: Scanner | undefined; + const invalidValueSentinel: object = {}; + + function getCookedText(kind: TemplateLiteralToken["kind"], rawText: string) { + if (!rawTextScanner) { + rawTextScanner = createScanner(ScriptTarget.Latest, /*skipTrivia*/ false, LanguageVariant.Standard); + } + switch (kind) { + case SyntaxKind.NoSubstitutionTemplateLiteral: + rawTextScanner.setText("`" + rawText + "`"); + break; + case SyntaxKind.TemplateHead: + // tslint:disable-next-line no-invalid-template-strings + rawTextScanner.setText("`" + rawText + "${"); + break; + case SyntaxKind.TemplateMiddle: + // tslint:disable-next-line no-invalid-template-strings + rawTextScanner.setText("}" + rawText + "${"); + break; + case SyntaxKind.TemplateTail: + rawTextScanner.setText("}" + rawText + "`"); + break; + } + + let token = rawTextScanner.scan(); + if (token === SyntaxKind.CloseBracketToken) { + token = rawTextScanner.reScanTemplateToken(); + } + + if (rawTextScanner.isUnterminated()) { + rawTextScanner.setText(undefined); + return invalidValueSentinel; + } + + let tokenValue: string | undefined; + switch (token) { + case SyntaxKind.NoSubstitutionTemplateLiteral: + case SyntaxKind.TemplateHead: + case SyntaxKind.TemplateMiddle: + case SyntaxKind.TemplateTail: + tokenValue = rawTextScanner.getTokenValue(); + break; + } + + if (rawTextScanner.scan() !== SyntaxKind.EndOfFileToken) { + rawTextScanner.setText(undefined); + return invalidValueSentinel; + } + + rawTextScanner.setText(undefined); + return tokenValue; + } + + function createTemplateLiteralLikeNode(kind: TemplateLiteralToken["kind"], text: string, rawText: string | undefined) { + const node = createSynthesizedNode(kind); + node.text = text; + if (rawText === undefined || text === rawText) { + node.rawText = rawText; + } + else { + const cooked = getCookedText(kind, rawText); + if (typeof cooked === "object") { + return Debug.fail("Invalid raw text"); + } + + Debug.assert(text === cooked, "Expected argument 'text' to be the normalized (i.e. 'cooked') version of argument 'rawText'."); + node.rawText = rawText; + } + return node; + } + + export function createTemplateHead(text: string, rawText?: string) { + const node = createTemplateLiteralLikeNode(SyntaxKind.TemplateHead, text, rawText); node.text = text; return node; } - export function createTemplateMiddle(text: string) { - const node = createSynthesizedNode(SyntaxKind.TemplateMiddle); + export function createTemplateMiddle(text: string, rawText?: string) { + const node = createTemplateLiteralLikeNode(SyntaxKind.TemplateMiddle, text, rawText); node.text = text; return node; } - export function createTemplateTail(text: string) { - const node = createSynthesizedNode(SyntaxKind.TemplateTail); + export function createTemplateTail(text: string, rawText?: string) { + const node = createTemplateLiteralLikeNode(SyntaxKind.TemplateTail, text, rawText); node.text = text; return node; } - export function createNoSubstitutionTemplateLiteral(text: string) { - const node = createSynthesizedNode(SyntaxKind.NoSubstitutionTemplateLiteral); - node.text = text; + export function createNoSubstitutionTemplateLiteral(text: string, rawText?: string) { + const node = createTemplateLiteralLikeNode(SyntaxKind.NoSubstitutionTemplateLiteral, text, rawText); return node; } diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts index 6451a4f5c96..3ce7f411e24 100644 --- a/src/compiler/parser.ts +++ b/src/compiler/parser.ts @@ -2300,9 +2300,19 @@ namespace ts { return fragment; } - function parseLiteralLikeNode(kind: SyntaxKind): LiteralExpression | LiteralLikeNode { - const node = createNode(kind); + function parseLiteralLikeNode(kind: SyntaxKind): LiteralLikeNode { + const node = createNode(kind); node.text = scanner.getTokenValue(); + switch (kind) { + case SyntaxKind.NoSubstitutionTemplateLiteral: + case SyntaxKind.TemplateHead: + case SyntaxKind.TemplateMiddle: + case SyntaxKind.TemplateTail: + const isLast = kind === SyntaxKind.NoSubstitutionTemplateLiteral || kind === SyntaxKind.TemplateTail; + const tokenText = scanner.getTokenText(); + (node).rawText = tokenText.substring(1, tokenText.length - (scanner.isUnterminated() ? 0 : isLast ? 1 : 2)); + break; + } if (scanner.hasExtendedUnicodeEscape()) { node.hasExtendedUnicodeEscape = true; diff --git a/src/compiler/transformers/es2015.ts b/src/compiler/transformers/es2015.ts index 793f6e1b1ef..b86db737879 100644 --- a/src/compiler/transformers/es2015.ts +++ b/src/compiler/transformers/es2015.ts @@ -3993,18 +3993,21 @@ namespace ts { * * @param node The ES6 template literal. */ - function getRawLiteral(node: LiteralLikeNode) { + function getRawLiteral(node: TemplateLiteralLikeNode) { // Find original source text, since we need to emit the raw strings of the tagged template. // The raw strings contain the (escaped) strings of what the user wrote. // Examples: `\n` is converted to "\\n", a template string with a newline to "\n". - let text = getSourceTextOfNodeFromSourceFile(currentSourceFile, node); + let text = node.rawText; + if (text === undefined) { + text = getSourceTextOfNodeFromSourceFile(currentSourceFile, node); - // text contains the original source, it will also contain quotes ("`"), dolar signs and braces ("${" and "}"), - // thus we need to remove those characters. - // First template piece starts with "`", others with "}" - // Last template piece ends with "`", others with "${" - const isLast = node.kind === SyntaxKind.NoSubstitutionTemplateLiteral || node.kind === SyntaxKind.TemplateTail; - text = text.substring(1, text.length - (isLast ? 1 : 2)); + // text contains the original source, it will also contain quotes ("`"), dolar signs and braces ("${" and "}"), + // thus we need to remove those characters. + // First template piece starts with "`", others with "}" + // Last template piece ends with "`", others with "${" + const isLast = node.kind === SyntaxKind.NoSubstitutionTemplateLiteral || node.kind === SyntaxKind.TemplateTail; + text = text.substring(1, text.length - (isLast ? 1 : 2)); + } // Newline normalization: // ES6 Spec 11.8.6.1 - Static Semantics of TV's and TRV's diff --git a/src/compiler/types.ts b/src/compiler/types.ts index a4373b7dc01..f50eb4d6399 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -1646,6 +1646,10 @@ namespace ts { hasExtendedUnicodeEscape?: boolean; } + export interface TemplateLiteralLikeNode extends LiteralLikeNode { + rawText?: string; + } + // The text property of a LiteralExpression stores the interpreted value of the literal in text form. For a StringLiteral, // or any literal of a template, this means quotes have been removed and escapes have been converted to actual characters. // For a NumericLiteral, the stored value is the toString() representation of the number. For example 1, 1.00, and 1e0 are all stored as just "1". @@ -1657,7 +1661,7 @@ namespace ts { kind: SyntaxKind.RegularExpressionLiteral; } - export interface NoSubstitutionTemplateLiteral extends LiteralExpression { + export interface NoSubstitutionTemplateLiteral extends LiteralExpression, TemplateLiteralLikeNode { kind: SyntaxKind.NoSubstitutionTemplateLiteral; } @@ -1696,17 +1700,17 @@ namespace ts { kind: SyntaxKind.BigIntLiteral; } - export interface TemplateHead extends LiteralLikeNode { + export interface TemplateHead extends TemplateLiteralLikeNode { kind: SyntaxKind.TemplateHead; parent: TemplateExpression; } - export interface TemplateMiddle extends LiteralLikeNode { + export interface TemplateMiddle extends TemplateLiteralLikeNode { kind: SyntaxKind.TemplateMiddle; parent: TemplateSpan; } - export interface TemplateTail extends LiteralLikeNode { + export interface TemplateTail extends TemplateLiteralLikeNode { kind: SyntaxKind.TemplateTail; parent: TemplateSpan; } diff --git a/src/compiler/utilities.ts b/src/compiler/utilities.ts index 0bb3c799af5..96acefdf135 100644 --- a/src/compiler/utilities.ts +++ b/src/compiler/utilities.ts @@ -566,8 +566,6 @@ namespace ts { return emitNode && emitNode.flags || 0; } - const escapeNoSubstitutionTemplateLiteralText = compose(escapeString, escapeTemplateSubstitution); - const escapeNonAsciiNoSubstitutionTemplateLiteralText = compose(escapeNonAsciiString, escapeTemplateSubstitution); export function getLiteralText(node: LiteralLikeNode, sourceFile: SourceFile, neverAsciiEscape: boolean | undefined) { // If we don't need to downlevel and we can reach the original source text using // the node's parent reference, then simply get the text as it was originally written. @@ -580,9 +578,7 @@ namespace ts { // If a NoSubstitutionTemplateLiteral appears to have a substitution in it, the original text // had to include a backslash: `not \${a} substitution`. - const escapeText = neverAsciiEscape || (getEmitFlags(node) & EmitFlags.NoAsciiEscaping) ? - node.kind === SyntaxKind.NoSubstitutionTemplateLiteral ? escapeNoSubstitutionTemplateLiteralText : escapeString : - node.kind === SyntaxKind.NoSubstitutionTemplateLiteral ? escapeNonAsciiNoSubstitutionTemplateLiteralText : escapeNonAsciiString; + const escapeText = neverAsciiEscape || (getEmitFlags(node) & EmitFlags.NoAsciiEscaping) ? escapeString : escapeNonAsciiString; // If we can't reach the original source text, use the canonical form if it's a number, // or a (possibly escaped) quoted form of the original text if it's string-like. @@ -595,15 +591,23 @@ namespace ts { return '"' + escapeText(node.text, CharacterCodes.doubleQuote) + '"'; } case SyntaxKind.NoSubstitutionTemplateLiteral: - return "`" + escapeText(node.text, CharacterCodes.backtick) + "`"; case SyntaxKind.TemplateHead: - // tslint:disable-next-line no-invalid-template-strings - return "`" + escapeText(node.text, CharacterCodes.backtick) + "${"; case SyntaxKind.TemplateMiddle: - // tslint:disable-next-line no-invalid-template-strings - return "}" + escapeText(node.text, CharacterCodes.backtick) + "${"; case SyntaxKind.TemplateTail: - return "}" + escapeText(node.text, CharacterCodes.backtick) + "`"; + const rawText = (node).rawText || escapeTemplateSubstitution(escapeText(node.text, CharacterCodes.backtick)); + switch (node.kind) { + case SyntaxKind.NoSubstitutionTemplateLiteral: + return "`" + rawText + "`"; + case SyntaxKind.TemplateHead: + // tslint:disable-next-line no-invalid-template-strings + return "`" + rawText + "${"; + case SyntaxKind.TemplateMiddle: + // tslint:disable-next-line no-invalid-template-strings + return "}" + rawText + "${"; + case SyntaxKind.TemplateTail: + return "}" + rawText + "`"; + } + break; case SyntaxKind.NumericLiteral: case SyntaxKind.BigIntLiteral: case SyntaxKind.RegularExpressionLiteral: @@ -3178,7 +3182,8 @@ namespace ts { // There is no reason for this other than that JSON.stringify does not handle it either. const doubleQuoteEscapedCharsRegExp = /[\\\"\u0000-\u001f\t\v\f\b\r\n\u2028\u2029\u0085]/g; const singleQuoteEscapedCharsRegExp = /[\\\'\u0000-\u001f\t\v\f\b\r\n\u2028\u2029\u0085]/g; - const backtickQuoteEscapedCharsRegExp = /[\\\`\u0000-\u001f\t\v\f\b\r\n\u2028\u2029\u0085]/g; + // Template strings should be preserved as much as possible + const backtickQuoteEscapedCharsRegExp = /[\\\`]/g; const escapedCharsMap = createMapFromTemplate({ "\t": "\\t", "\v": "\\v", diff --git a/src/harness/evaluator.ts b/src/harness/evaluator.ts index a22bdb958bc..c4cfc53069c 100644 --- a/src/harness/evaluator.ts +++ b/src/harness/evaluator.ts @@ -2,6 +2,7 @@ namespace evaluator { declare var Symbol: SymbolConstructor; const sourceFile = vpath.combine(vfs.srcFolder, "source.ts"); + const sourceFileJs = vpath.combine(vfs.srcFolder, "source.js"); function compile(sourceText: string, options?: ts.CompilerOptions) { const fs = vfs.createFromFileSystem(Harness.IO, /*ignoreCase*/ false); @@ -32,9 +33,8 @@ namespace evaluator { // Add "asyncIterator" if missing if (!ts.hasProperty(FakeSymbol, "asyncIterator")) Object.defineProperty(FakeSymbol, "asyncIterator", { value: Symbol.for("Symbol.asyncIterator"), configurable: true }); - function evaluate(result: compiler.CompilationResult, globals?: Record) { - globals = { Symbol: FakeSymbol, ...globals }; - + export function evaluateTypeScript(sourceText: string, options?: ts.CompilerOptions, globals?: Record) { + const result = compile(sourceText, options); if (ts.some(result.diagnostics)) { assert.ok(/*value*/ false, "Syntax error in evaluation source text:\n" + ts.formatDiagnostics(result.diagnostics, { getCanonicalFileName: file => file, @@ -46,6 +46,12 @@ namespace evaluator { const output = result.getOutput(sourceFile, "js")!; assert.isDefined(output); + return evaluateJavaScript(output.text, globals, output.file); + } + + export function evaluateJavaScript(sourceText: string, globals?: Record, sourceFile = sourceFileJs) { + globals = { Symbol: FakeSymbol, ...globals }; + const globalNames: string[] = []; const globalArgs: any[] = []; for (const name in globals) { @@ -55,15 +61,11 @@ namespace evaluator { } } - const evaluateText = `(function (module, exports, require, __dirname, __filename, ${globalNames.join(", ")}) { ${output.text} })`; - // tslint:disable-next-line:no-eval - const evaluateThunk = eval(evaluateText) as (module: any, exports: any, require: (id: string) => any, dirname: string, filename: string, ...globalArgs: any[]) => void; + const evaluateText = `(function (module, exports, require, __dirname, __filename, ${globalNames.join(", ")}) { ${sourceText} })`; + // tslint:disable-next-line:no-eval no-unused-expression + const evaluateThunk = (void 0, eval)(evaluateText) as (module: any, exports: any, require: (id: string) => any, dirname: string, filename: string, ...globalArgs: any[]) => void; const module: { exports: any; } = { exports: {} }; - evaluateThunk.call(globals, module, module.exports, noRequire, vpath.dirname(output.file), output.file, FakeSymbol, ...globalArgs); + evaluateThunk.call(globals, module, module.exports, noRequire, vpath.dirname(sourceFile), sourceFile, FakeSymbol, ...globalArgs); return module.exports; } - - export function evaluateTypeScript(sourceText: string, options?: ts.CompilerOptions, globals?: Record) { - return evaluate(compile(sourceText, options), globals); - } } \ No newline at end of file diff --git a/src/testRunner/unittests/transform.ts b/src/testRunner/unittests/transform.ts index 376b7d16ec8..35c3ab9a942 100644 --- a/src/testRunner/unittests/transform.ts +++ b/src/testRunner/unittests/transform.ts @@ -67,6 +67,24 @@ namespace ts { }); } + function testBaselineAndEvaluate(testName: string, test: () => string, onEvaluate: (exports: any) => void) { + describe(testName, () => { + let sourceText!: string; + before(() => { + sourceText = test(); + }); + after(() => { + sourceText = undefined!; + }); + it("compare baselines", () => { + Harness.Baseline.runBaseline(`transformApi/transformsCorrectly.${testName}.js`, sourceText); + }); + it("evaluate", () => { + onEvaluate(evaluator.evaluateJavaScript(sourceText)); + }); + }); + } + testBaseline("substitution", () => { return transformSourceFile(`var a = undefined;`, [replaceUndefinedWithVoid0]); }); @@ -440,6 +458,31 @@ namespace Foo { }); + testBaselineAndEvaluate("templateSpans", () => { + return transpileModule("const x = String.raw`\n\nhello`; exports.stringLength = x.trim().length;", { + compilerOptions: { + target: ScriptTarget.ESNext, + newLine: NewLineKind.CarriageReturnLineFeed, + }, + transformers: { + before: [transformSourceFile] + } + }).outputText; + + function transformSourceFile(context: TransformationContext): Transformer { + function visitor(node: Node): VisitResult { + if (isNoSubstitutionTemplateLiteral(node)) { + return createNoSubstitutionTemplateLiteral(node.text, node.rawText); + } + else { + return visitEachChild(node, visitor, context); + } + } + return sourceFile => visitNode(sourceFile, visitor, isSourceFile); + } + }, exports => { + assert.equal(exports.stringLength, 5); + }); }); } diff --git a/tests/baselines/reference/api/tsserverlibrary.d.ts b/tests/baselines/reference/api/tsserverlibrary.d.ts index 4c88e5fe219..c877d7d2d9a 100644 --- a/tests/baselines/reference/api/tsserverlibrary.d.ts +++ b/tests/baselines/reference/api/tsserverlibrary.d.ts @@ -997,13 +997,16 @@ declare namespace ts { isUnterminated?: boolean; hasExtendedUnicodeEscape?: boolean; } + export interface TemplateLiteralLikeNode extends LiteralLikeNode { + rawText?: string; + } export interface LiteralExpression extends LiteralLikeNode, PrimaryExpression { _literalExpressionBrand: any; } export interface RegularExpressionLiteral extends LiteralExpression { kind: SyntaxKind.RegularExpressionLiteral; } - export interface NoSubstitutionTemplateLiteral extends LiteralExpression { + export interface NoSubstitutionTemplateLiteral extends LiteralExpression, TemplateLiteralLikeNode { kind: SyntaxKind.NoSubstitutionTemplateLiteral; } export enum TokenFlags { @@ -1020,15 +1023,15 @@ declare namespace ts { export interface BigIntLiteral extends LiteralExpression { kind: SyntaxKind.BigIntLiteral; } - export interface TemplateHead extends LiteralLikeNode { + export interface TemplateHead extends TemplateLiteralLikeNode { kind: SyntaxKind.TemplateHead; parent: TemplateExpression; } - export interface TemplateMiddle extends LiteralLikeNode { + export interface TemplateMiddle extends TemplateLiteralLikeNode { kind: SyntaxKind.TemplateMiddle; parent: TemplateSpan; } - export interface TemplateTail extends LiteralLikeNode { + export interface TemplateTail extends TemplateLiteralLikeNode { kind: SyntaxKind.TemplateTail; parent: TemplateSpan; } @@ -3921,10 +3924,10 @@ declare namespace ts { function updateConditional(node: ConditionalExpression, condition: Expression, questionToken: Token, whenTrue: Expression, colonToken: Token, whenFalse: Expression): ConditionalExpression; function createTemplateExpression(head: TemplateHead, templateSpans: ReadonlyArray): TemplateExpression; function updateTemplateExpression(node: TemplateExpression, head: TemplateHead, templateSpans: ReadonlyArray): TemplateExpression; - function createTemplateHead(text: string): TemplateHead; - function createTemplateMiddle(text: string): TemplateMiddle; - function createTemplateTail(text: string): TemplateTail; - function createNoSubstitutionTemplateLiteral(text: string): NoSubstitutionTemplateLiteral; + function createTemplateHead(text: string, rawText?: string): TemplateHead; + function createTemplateMiddle(text: string, rawText?: string): TemplateMiddle; + function createTemplateTail(text: string, rawText?: string): TemplateTail; + function createNoSubstitutionTemplateLiteral(text: string, rawText?: string): NoSubstitutionTemplateLiteral; function createYield(expression?: Expression): YieldExpression; function createYield(asteriskToken: AsteriskToken | undefined, expression: Expression): YieldExpression; function updateYield(node: YieldExpression, asteriskToken: AsteriskToken | undefined, expression: Expression): YieldExpression; diff --git a/tests/baselines/reference/api/typescript.d.ts b/tests/baselines/reference/api/typescript.d.ts index bb30ccc908d..4c28abcbbc3 100644 --- a/tests/baselines/reference/api/typescript.d.ts +++ b/tests/baselines/reference/api/typescript.d.ts @@ -997,13 +997,16 @@ declare namespace ts { isUnterminated?: boolean; hasExtendedUnicodeEscape?: boolean; } + export interface TemplateLiteralLikeNode extends LiteralLikeNode { + rawText?: string; + } export interface LiteralExpression extends LiteralLikeNode, PrimaryExpression { _literalExpressionBrand: any; } export interface RegularExpressionLiteral extends LiteralExpression { kind: SyntaxKind.RegularExpressionLiteral; } - export interface NoSubstitutionTemplateLiteral extends LiteralExpression { + export interface NoSubstitutionTemplateLiteral extends LiteralExpression, TemplateLiteralLikeNode { kind: SyntaxKind.NoSubstitutionTemplateLiteral; } export enum TokenFlags { @@ -1020,15 +1023,15 @@ declare namespace ts { export interface BigIntLiteral extends LiteralExpression { kind: SyntaxKind.BigIntLiteral; } - export interface TemplateHead extends LiteralLikeNode { + export interface TemplateHead extends TemplateLiteralLikeNode { kind: SyntaxKind.TemplateHead; parent: TemplateExpression; } - export interface TemplateMiddle extends LiteralLikeNode { + export interface TemplateMiddle extends TemplateLiteralLikeNode { kind: SyntaxKind.TemplateMiddle; parent: TemplateSpan; } - export interface TemplateTail extends LiteralLikeNode { + export interface TemplateTail extends TemplateLiteralLikeNode { kind: SyntaxKind.TemplateTail; parent: TemplateSpan; } @@ -3921,10 +3924,10 @@ declare namespace ts { function updateConditional(node: ConditionalExpression, condition: Expression, questionToken: Token, whenTrue: Expression, colonToken: Token, whenFalse: Expression): ConditionalExpression; function createTemplateExpression(head: TemplateHead, templateSpans: ReadonlyArray): TemplateExpression; function updateTemplateExpression(node: TemplateExpression, head: TemplateHead, templateSpans: ReadonlyArray): TemplateExpression; - function createTemplateHead(text: string): TemplateHead; - function createTemplateMiddle(text: string): TemplateMiddle; - function createTemplateTail(text: string): TemplateTail; - function createNoSubstitutionTemplateLiteral(text: string): NoSubstitutionTemplateLiteral; + function createTemplateHead(text: string, rawText?: string): TemplateHead; + function createTemplateMiddle(text: string, rawText?: string): TemplateMiddle; + function createTemplateTail(text: string, rawText?: string): TemplateTail; + function createNoSubstitutionTemplateLiteral(text: string, rawText?: string): NoSubstitutionTemplateLiteral; function createYield(expression?: Expression): YieldExpression; function createYield(asteriskToken: AsteriskToken | undefined, expression: Expression): YieldExpression; function updateYield(node: YieldExpression, asteriskToken: AsteriskToken | undefined, expression: Expression): YieldExpression; diff --git a/tests/baselines/reference/transformApi/transformsCorrectly.templateSpans.js b/tests/baselines/reference/transformApi/transformsCorrectly.templateSpans.js new file mode 100644 index 00000000000..00c8f992444 --- /dev/null +++ b/tests/baselines/reference/transformApi/transformsCorrectly.templateSpans.js @@ -0,0 +1,4 @@ +const x = String.raw ` + +hello`; +exports.stringLength = x.trim().length;