From 2f624f5df369eb3bf245949953212cce7040887a Mon Sep 17 00:00:00 2001 From: Ron Buckton Date: Tue, 7 Feb 2017 14:36:15 -0800 Subject: [PATCH] Expose transformations as public API --- Gulpfile.ts | 9 +- Jakefile.js | 39 +++++-- src/compiler/comments.ts | 106 ++++++++++++++---- src/compiler/emitter.ts | 10 +- src/compiler/factory.ts | 32 ++++++ src/compiler/parser.ts | 6 +- src/compiler/program.ts | 10 +- src/compiler/scanner.ts | 16 +-- src/compiler/transformer.ts | 22 +++- src/compiler/transformers/generators.ts | 2 +- src/compiler/transformers/ts.ts | 11 +- src/compiler/types.ts | 40 +++++-- src/harness/tsconfig.json | 6 +- src/harness/unittests/customTransforms.ts | 86 ++++++++++++++ src/harness/unittests/transform.ts | 43 +++++++ src/services/outliningElementsCollector.ts | 4 +- src/services/services.ts | 3 +- src/services/transform.ts | 34 ++++++ src/services/transpile.ts | 3 +- src/services/tsconfig.json | 1 + src/services/types.ts | 5 + .../reference/customTransforms/after.js | 15 +++ .../reference/customTransforms/before.js | 15 +++ .../reference/customTransforms/both.js | 17 +++ .../transformsCorrectly.substitution.js | 1 + 25 files changed, 459 insertions(+), 77 deletions(-) create mode 100644 src/harness/unittests/customTransforms.ts create mode 100644 src/harness/unittests/transform.ts create mode 100644 src/services/transform.ts create mode 100644 tests/baselines/reference/customTransforms/after.js create mode 100644 tests/baselines/reference/customTransforms/before.js create mode 100644 tests/baselines/reference/customTransforms/both.js create mode 100644 tests/baselines/reference/transformApi/transformsCorrectly.substitution.js diff --git a/Gulpfile.ts b/Gulpfile.ts index 4f205bfb0ff..314ca4a07c4 100644 --- a/Gulpfile.ts +++ b/Gulpfile.ts @@ -41,7 +41,7 @@ const {runTestsInParallel} = mochaParallel; Error.stackTraceLimit = 1000; const cmdLineOptions = minimist(process.argv.slice(2), { - boolean: ["debug", "light", "colors", "lint", "soft"], + boolean: ["debug", "inspect", "light", "colors", "lint", "soft"], string: ["browser", "tests", "host", "reporter", "stackTraceLimit"], alias: { d: "debug", @@ -57,6 +57,7 @@ const cmdLineOptions = minimist(process.argv.slice(2), { soft: false, colors: process.env.colors || process.env.color || true, debug: process.env.debug || process.env.d, + inspect: process.env.inspect, host: process.env.TYPESCRIPT_HOST || process.env.host || "node", browser: process.env.browser || process.env.b || "IE", tests: process.env.test || process.env.tests || process.env.t, @@ -588,6 +589,7 @@ function runConsoleTests(defaultReporter: string, runInParallel: boolean, done: cleanTestDirs((err) => { if (err) { console.error(err); failWithStatus(err, 1); } const debug = cmdLineOptions["debug"]; + const inspect = cmdLineOptions["inspect"]; const tests = cmdLineOptions["tests"]; const light = cmdLineOptions["light"]; const stackTraceLimit = cmdLineOptions["stackTraceLimit"]; @@ -624,7 +626,10 @@ function runConsoleTests(defaultReporter: string, runInParallel: boolean, done: // default timeout is 2sec which really should be enough, but maybe we just need a small amount longer if (!runInParallel) { const args = []; - if (debug) { + if (inspect) { + args.push("--inspect"); + } + if (inspect || debug) { args.push("--debug-brk"); } args.push("-R", reporter); diff --git a/Jakefile.js b/Jakefile.js index 0b3a26dbb77..8d58fd4d3f3 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -150,6 +150,7 @@ var servicesSources = [ "shims.ts", "signatureHelp.ts", "symbolDisplay.ts", + "transform.ts", "transpile.ts", // Formatting "formatting/formatting.ts", @@ -270,6 +271,8 @@ var harnessSources = harnessCoreSources.concat([ "matchFiles.ts", "initializeTSConfig.ts", "printer.ts", + "transform.ts", + "customTransforms.ts", ].map(function (f) { return path.join(unittestsDirectory, f); })).concat([ @@ -919,6 +922,7 @@ function runConsoleTests(defaultReporter, runInParallel) { } var debug = process.env.debug || process.env.d; + var inspect = process.env.inspect; tests = process.env.test || process.env.tests || process.env.t; var light = process.env.light || false; var stackTraceLimit = process.env.stackTraceLimit; @@ -948,18 +952,39 @@ function runConsoleTests(defaultReporter, runInParallel) { testTimeout = 800000; } - colors = process.env.colors || process.env.color; - colors = colors ? ' --no-colors ' : ' --colors '; - reporter = process.env.reporter || process.env.r || defaultReporter; - var bail = (process.env.bail || process.env.b) ? "--bail" : ""; + var colors = process.env.colors || process.env.color || true; + var reporter = process.env.reporter || process.env.r || defaultReporter; + var bail = process.env.bail || process.env.b; var lintFlag = process.env.lint !== 'false'; // timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally // default timeout is 2sec which really should be enough, but maybe we just need a small amount longer if (!runInParallel) { var startTime = mark(); - tests = tests ? ' -g "' + tests + '"' : ''; - var cmd = "mocha" + (debug ? " --debug-brk" : "") + " -R " + reporter + tests + colors + bail + ' -t ' + testTimeout + ' ' + run; + var args = []; + if (inspect) { + args.push("--inspect"); + } + if (inspect || debug) { + args.push("--debug-brk"); + } + args.push("-R", reporter); + if (tests) { + args.push("-g", `"${tests}"`); + } + if (colors) { + args.push("--colors"); + } + else { + args.push("--no-colors"); + } + if (bail) { + args.push("--bail"); + } + args.push("-t", testTimeout); + args.push(run); + + var cmd = "mocha " + args.join(" "); console.log(cmd); var savedNodeEnv = process.env.NODE_ENV; @@ -980,7 +1005,7 @@ function runConsoleTests(defaultReporter, runInParallel) { var savedNodeEnv = process.env.NODE_ENV; process.env.NODE_ENV = "development"; var startTime = mark(); - runTestsInParallel(taskConfigsFolder, run, { testTimeout: testTimeout, noColors: colors === " --no-colors " }, function (err) { + runTestsInParallel(taskConfigsFolder, run, { testTimeout: testTimeout, noColors: !colors }, function (err) { process.env.NODE_ENV = savedNodeEnv; measure(startTime); // last worker clean everything and runs linter in case if there were no errors diff --git a/src/compiler/comments.ts b/src/compiler/comments.ts index 200158c2f5c..49ed0986a55 100644 --- a/src/compiler/comments.ts +++ b/src/compiler/comments.ts @@ -41,18 +41,14 @@ namespace ts { } if (node) { - const { pos, end } = getCommentRange(node); - const emitFlags = getEmitFlags(node); + hasWrittenComment = false; + + const emitNode = node.emitNode; + const emitFlags = emitNode && emitNode.flags; + const { pos, end } = emitNode && emitNode.commentRange || node; if ((pos < 0 && end < 0) || (pos === end)) { // Both pos and end are synthesized, so just emit the node without comments. - if (emitFlags & EmitFlags.NoNestedComments) { - disabled = true; - emitCallback(hint, node); - disabled = false; - } - else { - emitCallback(hint, node); - } + emitNodeWithSynthesizedComments(hint, node, emitNode, emitFlags, emitCallback); } else { if (extendedDiagnostics) { @@ -92,17 +88,10 @@ namespace ts { performance.measure("commentTime", "preEmitNodeWithComment"); } - if (emitFlags & EmitFlags.NoNestedComments) { - disabled = true; - emitCallback(hint, node); - disabled = false; - } - else { - emitCallback(hint, node); - } + emitNodeWithSynthesizedComments(hint, node, emitNode, emitFlags, emitCallback); if (extendedDiagnostics) { - performance.mark("beginEmitNodeWithComment"); + performance.mark("postEmitNodeWithComment"); } // Restore previous container state. @@ -117,12 +106,89 @@ namespace ts { } if (extendedDiagnostics) { - performance.measure("commentTime", "beginEmitNodeWithComment"); + performance.measure("commentTime", "postEmitNodeWithComment"); } } } } + function emitNodeWithSynthesizedComments(hint: EmitHint, node: Node, emitNode: EmitNode, emitFlags: EmitFlags, emitCallback: (hint: EmitHint, node: Node) => void) { + const leadingComments = emitNode && emitNode.leadingComments; + if (some(leadingComments)) { + if (extendedDiagnostics) { + performance.mark("preEmitNodeWithSynthesizedComments"); + } + + forEach(leadingComments, emitLeadingSynthesizedComment); + + if (extendedDiagnostics) { + performance.measure("commentTime", "preEmitNodeWithSynthesizedComments"); + } + } + + emitNodeWithNestedComments(hint, node, emitFlags, emitCallback); + + const trailingComments = emitNode && emitNode.trailingComments; + if (some(trailingComments)) { + if (extendedDiagnostics) { + performance.mark("postEmitNodeWithSynthesizedComments"); + } + + debugger; + forEach(trailingComments, emitTrailingSynthesizedComment); + + if (extendedDiagnostics) { + performance.measure("commentTime", "postEmitNodeWithSynthesizedComments"); + } + } + } + + function emitLeadingSynthesizedComment(comment: SynthesizedComment) { + if (comment.kind === SyntaxKind.SingleLineCommentTrivia) { + writer.writeLine(); + } + writeSynthesizedComment(comment); + if (comment.hasTrailingNewLine || comment.kind === SyntaxKind.SingleLineCommentTrivia) { + writer.writeLine(); + } + else { + writer.write(" "); + } + } + + function emitTrailingSynthesizedComment(comment: SynthesizedComment) { + if (!writer.isAtStartOfLine()) { + writer.write(" "); + } + writeSynthesizedComment(comment); + if (comment.hasTrailingNewLine) { + writer.writeLine(); + } + } + + function writeSynthesizedComment(comment: SynthesizedComment) { + const text = formatSynthesizedComment(comment); + const lineMap = comment.kind === SyntaxKind.MultiLineCommentTrivia ? computeLineStarts(text) : undefined; + writeCommentRange(text, lineMap, writer, 0, text.length, newLine); + } + + function formatSynthesizedComment(comment: SynthesizedComment) { + return comment.kind === SyntaxKind.MultiLineCommentTrivia + ? `/*${comment.text}*/` + : `//${comment.text}`; + } + + function emitNodeWithNestedComments(hint: EmitHint, node: Node, emitFlags: EmitFlags, emitCallback: (hint: EmitHint, node: Node) => void) { + if (emitFlags & EmitFlags.NoNestedComments) { + disabled = true; + emitCallback(hint, node); + disabled = false; + } + else { + emitCallback(hint, node); + } + } + function emitBodyWithDetachedComments(node: Node, detachedRange: TextRange, emitCallback: (node: Node) => void) { if (extendedDiagnostics) { performance.mark("preEmitBodyWithDetachedComments"); diff --git a/src/compiler/emitter.ts b/src/compiler/emitter.ts index 700102a7069..96c87b2efd0 100644 --- a/src/compiler/emitter.ts +++ b/src/compiler/emitter.ts @@ -10,14 +10,13 @@ namespace ts { /*@internal*/ // targetSourceFile is when users only want one file in entire project to be emitted. This is used in compileOnSave feature - export function emitFiles(resolver: EmitResolver, host: EmitHost, targetSourceFile: SourceFile, emitOnlyDtsFiles?: boolean): EmitResult { + export function emitFiles(resolver: EmitResolver, host: EmitHost, targetSourceFile: SourceFile, emitOnlyDtsFiles?: boolean, transformers?: Transformer[]): EmitResult { const compilerOptions = host.getCompilerOptions(); const moduleKind = getEmitModuleKind(compilerOptions); const sourceMapDataList: SourceMapData[] = compilerOptions.sourceMap || compilerOptions.inlineSourceMap ? [] : undefined; const emittedFilesList: string[] = compilerOptions.listEmittedFiles ? [] : undefined; const emitterDiagnostics = createDiagnosticCollection(); const newLine = host.getNewLine(); - const transformers = emitOnlyDtsFiles ? [] : getTransformers(compilerOptions); const writer = createTextWriter(newLine); const sourceMap = createSourceMapWriter(host, writer); @@ -56,9 +55,7 @@ namespace ts { performance.measure("printTime", "beforePrint"); // Clean up emit nodes on parse tree - for (const sourceFile of sourceFiles) { - disposeEmitNodes(sourceFile); - } + transform.dispose(); return { emitSkipped, @@ -756,9 +753,6 @@ namespace ts { // SyntaxKind.NumericLiteral function emitNumericLiteral(node: NumericLiteral) { emitLiteral(node); - if (node.trailingComment) { - write(` /*${node.trailingComment}*/`); - } } // SyntaxKind.StringLiteral diff --git a/src/compiler/factory.ts b/src/compiler/factory.ts index d5d8482a0ba..95821ba10de 100644 --- a/src/compiler/factory.ts +++ b/src/compiler/factory.ts @@ -1906,6 +1906,34 @@ namespace ts { return node; } + export function getSyntheticLeadingComments(node: Node): SynthesizedComment[] | undefined { + const emitNode = node.emitNode; + return emitNode && emitNode.leadingComments; + } + + export function setSyntheticLeadingComments(node: T, comments: SynthesizedComment[]) { + getOrCreateEmitNode(node).leadingComments = comments; + return node; + } + + export function addSyntheticLeadingComment(node: T, kind: SyntaxKind.SingleLineCommentTrivia | SyntaxKind.MultiLineCommentTrivia, text: string, hasTrailingNewLine?: boolean) { + return setSyntheticLeadingComments(node, append(getSyntheticLeadingComments(node), { kind, pos: -1, end: -1, hasTrailingNewLine, text })); + } + + export function getSyntheticTrailingComments(node: Node): SynthesizedComment[] | undefined { + const emitNode = node.emitNode; + return emitNode && emitNode.trailingComments; + } + + export function setSyntheticTrailingComments(node: T, comments: SynthesizedComment[]) { + getOrCreateEmitNode(node).trailingComments = comments; + return node; + } + + export function addSyntheticTrailingComment(node: T, kind: SyntaxKind.SingleLineCommentTrivia | SyntaxKind.MultiLineCommentTrivia, text: string, hasTrailingNewLine?: boolean) { + return setSyntheticTrailingComments(node, append(getSyntheticTrailingComments(node), { kind, pos: -1, end: -1, hasTrailingNewLine, text })); + } + /** * Gets the constant value to emit for an expression. */ @@ -2018,6 +2046,8 @@ namespace ts { function mergeEmitNode(sourceEmitNode: EmitNode, destEmitNode: EmitNode) { const { flags, + leadingComments, + trailingComments, commentRange, sourceMapRange, tokenSourceMapRanges, @@ -2025,6 +2055,8 @@ namespace ts { helpers } = sourceEmitNode; if (!destEmitNode) destEmitNode = {}; + if (leadingComments) destEmitNode.leadingComments = addRange(leadingComments.slice(), destEmitNode.leadingComments); + if (trailingComments) destEmitNode.trailingComments = addRange(trailingComments.slice(), destEmitNode.trailingComments); if (flags) destEmitNode.flags = flags; if (commentRange) destEmitNode.commentRange = commentRange; if (sourceMapRange) destEmitNode.sourceMapRange = sourceMapRange; diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts index c5439aa051b..51887474396 100644 --- a/src/compiler/parser.ts +++ b/src/compiler/parser.ts @@ -5816,7 +5816,11 @@ namespace ts { } } - const range = { pos: triviaScanner.getTokenPos(), end: triviaScanner.getTextPos(), kind: triviaScanner.getToken() }; + const range = { + kind: triviaScanner.getToken(), + pos: triviaScanner.getTokenPos(), + end: triviaScanner.getTextPos(), + }; const comment = sourceText.substring(range.pos, range.end); const referencePathMatchResult = getFileReferenceFromReferencePath(comment, range); diff --git a/src/compiler/program.ts b/src/compiler/program.ts index 62f3c066442..eb9f0f0e0a1 100644 --- a/src/compiler/program.ts +++ b/src/compiler/program.ts @@ -754,15 +754,15 @@ namespace ts { return noDiagnosticsTypeChecker || (noDiagnosticsTypeChecker = createTypeChecker(program, /*produceDiagnostics:*/ false)); } - function emit(sourceFile?: SourceFile, writeFileCallback?: WriteFileCallback, cancellationToken?: CancellationToken, emitOnlyDtsFiles?: boolean): EmitResult { - return runWithCancellationToken(() => emitWorker(program, sourceFile, writeFileCallback, cancellationToken, emitOnlyDtsFiles)); + function emit(sourceFile?: SourceFile, writeFileCallback?: WriteFileCallback, cancellationToken?: CancellationToken, emitOnlyDtsFiles?: boolean, transformers?: CustomTransformers): EmitResult { + return runWithCancellationToken(() => emitWorker(program, sourceFile, writeFileCallback, cancellationToken, emitOnlyDtsFiles, transformers)); } function isEmitBlocked(emitFileName: string): boolean { return hasEmitBlockingDiagnostics.contains(toPath(emitFileName, currentDirectory, getCanonicalFileName)); } - function emitWorker(program: Program, sourceFile: SourceFile, writeFileCallback: WriteFileCallback, cancellationToken: CancellationToken, emitOnlyDtsFiles?: boolean): EmitResult { + function emitWorker(program: Program, sourceFile: SourceFile, writeFileCallback: WriteFileCallback, cancellationToken: CancellationToken, emitOnlyDtsFiles?: boolean, customTransformers?: CustomTransformers): EmitResult { let declarationDiagnostics: Diagnostic[] = []; if (options.noEmit) { @@ -804,11 +804,13 @@ namespace ts { performance.mark("beforeEmit"); + const transformers = emitOnlyDtsFiles ? [] : getTransformers(options, customTransformers); const emitResult = emitFiles( emitResolver, getEmitHost(writeFileCallback), sourceFile, - emitOnlyDtsFiles); + emitOnlyDtsFiles, + transformers); performance.mark("afterEmit"); performance.measure("Emit", "beforeEmit", "afterEmit"); diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts index 52c75a37005..fb91b0ff5b7 100644 --- a/src/compiler/scanner.ts +++ b/src/compiler/scanner.ts @@ -608,10 +608,10 @@ namespace ts { * @returns If "reduce" is true, the accumulated value. If "reduce" is false, the first truthy * return value of the callback. */ - function iterateCommentRanges(reduce: boolean, text: string, pos: number, trailing: boolean, cb: (pos: number, end: number, kind: SyntaxKind, hasTrailingNewLine: boolean, state: T, memo: U) => U, state: T, initial?: U): U { + function iterateCommentRanges(reduce: boolean, text: string, pos: number, trailing: boolean, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T, memo: U) => U, state: T, initial?: U): U { let pendingPos: number; let pendingEnd: number; - let pendingKind: SyntaxKind; + let pendingKind: CommentKind; let pendingHasTrailingNewLine: boolean; let hasPendingCommentRange = false; let collecting = trailing || pos === 0; @@ -707,28 +707,28 @@ namespace ts { return accumulator; } - export function forEachLeadingCommentRange(text: string, pos: number, cb: (pos: number, end: number, kind: SyntaxKind, hasTrailingNewLine: boolean, state: T) => U, state?: T) { + export function forEachLeadingCommentRange(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T) => U, state?: T) { return iterateCommentRanges(/*reduce*/ false, text, pos, /*trailing*/ false, cb, state); } - export function forEachTrailingCommentRange(text: string, pos: number, cb: (pos: number, end: number, kind: SyntaxKind, hasTrailingNewLine: boolean, state: T) => U, state?: T) { + export function forEachTrailingCommentRange(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T) => U, state?: T) { return iterateCommentRanges(/*reduce*/ false, text, pos, /*trailing*/ true, cb, state); } - export function reduceEachLeadingCommentRange(text: string, pos: number, cb: (pos: number, end: number, kind: SyntaxKind, hasTrailingNewLine: boolean, state: T, memo: U) => U, state: T, initial: U) { + export function reduceEachLeadingCommentRange(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T, memo: U) => U, state: T, initial: U) { return iterateCommentRanges(/*reduce*/ true, text, pos, /*trailing*/ false, cb, state, initial); } - export function reduceEachTrailingCommentRange(text: string, pos: number, cb: (pos: number, end: number, kind: SyntaxKind, hasTrailingNewLine: boolean, state: T, memo: U) => U, state: T, initial: U) { + export function reduceEachTrailingCommentRange(text: string, pos: number, cb: (pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, state: T, memo: U) => U, state: T, initial: U) { return iterateCommentRanges(/*reduce*/ true, text, pos, /*trailing*/ true, cb, state, initial); } - function appendCommentRange(pos: number, end: number, kind: SyntaxKind, hasTrailingNewLine: boolean, _state: any, comments: CommentRange[]) { + function appendCommentRange(pos: number, end: number, kind: CommentKind, hasTrailingNewLine: boolean, _state: any, comments: CommentRange[]) { if (!comments) { comments = []; } - comments.push({ pos, end, hasTrailingNewLine, kind }); + comments.push({ kind, pos, end, hasTrailingNewLine }); return comments; } diff --git a/src/compiler/transformer.ts b/src/compiler/transformer.ts index 62469df79c1..68f149c666f 100644 --- a/src/compiler/transformer.ts +++ b/src/compiler/transformer.ts @@ -29,12 +29,14 @@ namespace ts { EmitNotifications = 1 << 1, } - export function getTransformers(compilerOptions: CompilerOptions) { + export function getTransformers(compilerOptions: CompilerOptions, customTransformers?: CustomTransformers) { const jsx = compilerOptions.jsx; const languageVersion = getEmitScriptTarget(compilerOptions); const moduleKind = getEmitModuleKind(compilerOptions); const transformers: Transformer[] = []; + addRange(transformers, customTransformers && customTransformers.before); + transformers.push(transformTypeScript); if (jsx === JsxEmit.React) { @@ -66,6 +68,8 @@ namespace ts { transformers.push(transformES5); } + addRange(transformers, customTransformers && customTransformers.after); + return transformers; } @@ -79,16 +83,13 @@ namespace ts { */ export function transformFiles(resolver: EmitResolver, host: EmitHost, sourceFiles: SourceFile[], transformers: Transformer[]): TransformationResult { const enabledSyntaxKindFeatures = new Array(SyntaxKind.Count); - let lexicalEnvironmentDisabled = false; - let lexicalEnvironmentVariableDeclarations: VariableDeclaration[]; let lexicalEnvironmentFunctionDeclarations: FunctionDeclaration[]; let lexicalEnvironmentVariableDeclarationsStack: VariableDeclaration[][] = []; let lexicalEnvironmentFunctionDeclarationsStack: FunctionDeclaration[][] = []; let lexicalEnvironmentStackOffset = 0; let lexicalEnvironmentSuspended = false; - let emitHelpers: EmitHelper[]; // The transformation context is provided to each transformer as part of transformer @@ -113,6 +114,9 @@ namespace ts { isEmitNotificationEnabled }; + // Ensure the parse tree is clean before applying transformations + dispose(); + performance.mark("beforeTransform"); // Chain together and initialize each transformer. @@ -130,7 +134,8 @@ namespace ts { return { transformed, emitNodeWithSubstitution, - emitNodeWithNotification + emitNodeWithNotification, + dispose }; /** @@ -323,5 +328,12 @@ namespace ts { emitHelpers = undefined; return helpers; } + + function dispose() { + // Clean up emit nodes on parse tree + for (const sourceFile of sourceFiles) { + disposeEmitNodes(sourceFile); + } + } } } diff --git a/src/compiler/transformers/generators.ts b/src/compiler/transformers/generators.ts index cabd5d2a96d..f40c2994998 100644 --- a/src/compiler/transformers/generators.ts +++ b/src/compiler/transformers/generators.ts @@ -2419,7 +2419,7 @@ namespace ts { */ function createInstruction(instruction: Instruction): NumericLiteral { const literal = createLiteral(instruction); - literal.trailingComment = getInstructionName(instruction); + addSyntheticTrailingComment(literal, SyntaxKind.MultiLineCommentTrivia, getInstructionName(instruction)); return literal; } diff --git a/src/compiler/transformers/ts.ts b/src/compiler/transformers/ts.ts index 4ef3bd5c067..c28d74f8527 100644 --- a/src/compiler/transformers/ts.ts +++ b/src/compiler/transformers/ts.ts @@ -3312,17 +3312,20 @@ namespace ts { function substituteConstantValue(node: PropertyAccessExpression | ElementAccessExpression): LeftHandSideExpression { const constantValue = tryGetConstEnumValue(node); if (constantValue !== undefined) { + // track the constant value on the node for the printer in needsDotDotForPropertyAccess + setConstantValue(node, constantValue); + const substitute = createLiteral(constantValue); - setSourceMapRange(substitute, node); - setCommentRange(substitute, node); if (!compilerOptions.removeComments) { const propertyName = isPropertyAccessExpression(node) ? declarationNameToString(node.name) : getTextOfNode(node.argumentExpression); - substitute.trailingComment = ` ${propertyName} `; + + addSyntheticTrailingComment(substitute, SyntaxKind.MultiLineCommentTrivia, ` ${propertyName} `); + // wrap the substituted node so that it emits its own comments. + return createPartiallyEmittedExpression(substitute); } - setConstantValue(node, constantValue); return substitute; } diff --git a/src/compiler/types.ts b/src/compiler/types.ts index 49e744682bc..275760bfc6a 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -1313,7 +1313,6 @@ export interface NumericLiteral extends LiteralExpression { kind: SyntaxKind.NumericLiteral; - trailingComment?: string; } export interface TemplateHead extends LiteralLikeNode { @@ -1893,9 +1892,17 @@ fileName: string; } + export type CommentKind = SyntaxKind.SingleLineCommentTrivia | SyntaxKind.MultiLineCommentTrivia; + export interface CommentRange extends TextRange { hasTrailingNewLine?: boolean; - kind: SyntaxKind; + kind: CommentKind; + } + + export interface SynthesizedComment extends CommentRange { + text: string; + pos: -1; + end: -1; } // represents a top level: { type } expression in a JSDoc comment. @@ -2265,7 +2272,7 @@ * used for writing the JavaScript and declaration files. Otherwise, the writeFile parameter * will be invoked when writing the JavaScript and declaration files. */ - emit(targetSourceFile?: SourceFile, writeFile?: WriteFileCallback, cancellationToken?: CancellationToken, emitOnlyDtsFiles?: boolean): EmitResult; + emit(targetSourceFile?: SourceFile, writeFile?: WriteFileCallback, cancellationToken?: CancellationToken, emitOnlyDtsFiles?: boolean, customTransformers?: CustomTransformers): EmitResult; getOptionsDiagnostics(cancellationToken?: CancellationToken): Diagnostic[]; getGlobalDiagnostics(cancellationToken?: CancellationToken): Diagnostic[]; @@ -2299,6 +2306,13 @@ /* @internal */ structureIsReused?: boolean; } + export interface CustomTransformers { + /** Custom transformers to evaluate before built-in transformations. */ + before?: Transformer[]; + /** Custom transformers to evaluate after built-in transformations. */ + after?: Transformer[]; + } + export interface SourceMapSpan { /** Line number in the .js file. */ emittedLine: number; @@ -3724,9 +3738,11 @@ export interface EmitNode { annotatedNodes?: Node[]; // Tracks Parse-tree nodes with EmitNodes for eventual cleanup. flags?: EmitFlags; // Flags that customize emit + leadingComments?: SynthesizedComment[]; // Synthesized leading comments + trailingComments?: SynthesizedComment[]; // Synthesized trailing comments commentRange?: TextRange; // The text range to use when emitting leading or trailing comments sourceMapRange?: TextRange; // The text range to use when emitting leading or trailing source mappings - tokenSourceMapRanges?: TextRange[]; // The text range to use when emitting source mappings for tokens + tokenSourceMapRanges?: TextRange[]; // The text range to use when emitting source mappings for tokens constantValue?: number; // The constant value of an expression externalHelpersModuleName?: Identifier; // The local name for an imported helpers module helpers?: EmitHelper[]; // Emit helpers for the node @@ -3762,7 +3778,7 @@ export interface EmitHelper { readonly name: string; // A unique name for this helper. - readonly scoped: boolean; // Indicates whether ther helper MUST be emitted in the current scope. + readonly scoped: boolean; // Indicates whether the helper MUST be emitted in the current scope. readonly text: string; // ES3-compatible raw script text. readonly priority?: number; // Helpers with a higher priority are emitted earlier than other helpers on the node. } @@ -3809,11 +3825,10 @@ writeFile: WriteFileCallback; } - /* @internal */ export interface TransformationContext { - getCompilerOptions(): CompilerOptions; - getEmitResolver(): EmitResolver; - getEmitHost(): EmitHost; + /*@internal*/ getCompilerOptions(): CompilerOptions; + /*@internal*/ getEmitResolver(): EmitResolver; + /*@internal*/ getEmitHost(): EmitHost; /** Starts a new lexical environment. */ startLexicalEnvironment(): void; @@ -3882,7 +3897,6 @@ onEmitNode?: (hint: EmitHint, node: Node, emitCallback: (hint: EmitHint, node: Node) => void) => void; } - /* @internal */ export interface TransformationResult { /** * Gets the transformed source files. @@ -3906,9 +3920,13 @@ * @param emitCallback A callback used to emit the node. */ emitNodeWithNotification(hint: EmitHint, node: Node, emitCallback: (hint: EmitHint, node: Node) => void): void; + + /** + * Clean up EmitNode entries on any parse-tree nodes. + */ + dispose(): void; } - /* @internal */ export type Transformer = (context: TransformationContext) => (node: SourceFile) => SourceFile; export interface Printer { diff --git a/src/harness/tsconfig.json b/src/harness/tsconfig.json index bea688d358b..32af0eb2601 100644 --- a/src/harness/tsconfig.json +++ b/src/harness/tsconfig.json @@ -77,8 +77,8 @@ "../services/codefixes/helpers.ts", "../services/codefixes/importFixes.ts", "../services/codefixes/unusedIdentifierFixes.ts", - "../services/harness.ts", + "harness.ts", "sourceMapRecorder.ts", "harnessLanguageService.ts", "fourslash.ts", @@ -119,6 +119,8 @@ "./unittests/compileOnSave.ts", "./unittests/typingsInstaller.ts", "./unittests/projectErrors.ts", - "./unittests/printer.ts" + "./unittests/printer.ts", + "./unittests/transform.ts", + "./unittests/customTransforms.ts" ] } diff --git a/src/harness/unittests/customTransforms.ts b/src/harness/unittests/customTransforms.ts new file mode 100644 index 00000000000..af8db28b69b --- /dev/null +++ b/src/harness/unittests/customTransforms.ts @@ -0,0 +1,86 @@ +/// +/// + +namespace ts { + describe("customTransforms", () => { + function emitsCorrectly(name: string, sources: { file: string, text: string }[], customTransformers: CustomTransformers) { + it(name, () => { + const roots = sources.map(source => createSourceFile(source.file, source.text, ScriptTarget.ES2015)); + const fileMap = arrayToMap(roots, file => file.fileName); + const outputs = createMap(); + const options: CompilerOptions = {}; + const host: CompilerHost = { + getSourceFile: (fileName) => fileMap.get(fileName), + getDefaultLibFileName: () => "lib.d.ts", + getCurrentDirectory: () => "", + getDirectories: () => [], + getCanonicalFileName: (fileName) => fileName, + useCaseSensitiveFileNames: () => true, + getNewLine: () => "\n", + fileExists: (fileName) => fileMap.has(fileName), + readFile: (fileName) => fileMap.has(fileName) ? fileMap.get(fileName).text : undefined, + writeFile: (fileName, text) => outputs.set(fileName, text), + }; + + const program = createProgram(arrayFrom(fileMap.keys()), options, host); + program.emit(/*targetSourceFile*/ undefined, host.writeFile, /*cancellationToken*/ undefined, /*emitOnlyDtsFiles*/ false, customTransformers); + Harness.Baseline.runBaseline(`customTransforms/${name}.js`, () => { + let content = ""; + for (const [file, text] of arrayFrom(outputs.entries())) { + if (content) content += "\n\n"; + content += `// [${file}]\n`; + content += text; + } + return content; + }); + }); + } + + const sources = [{ + file: "source.ts", + text: ` + function f1() { } + class c() { } + enum e { } + // leading + function f2() { } // trailing + ` + }]; + + const before: Transformer = context => { + return file => visitEachChild(file, visit, context); + function visit(node: Node): VisitResult { + switch (node.kind) { + case SyntaxKind.FunctionDeclaration: + return visitFunction(node); + default: + return visitEachChild(node, visit, context); + } + } + function visitFunction(node: FunctionDeclaration) { + addSyntheticLeadingComment(node, SyntaxKind.MultiLineCommentTrivia, "@before", /*hasTrailingNewLine*/ true); + return node; + } + }; + + const after: Transformer = context => { + return file => visitEachChild(file, visit, context); + function visit(node: Node): VisitResult { + switch (node.kind) { + case SyntaxKind.VariableStatement: + return visitVariableStatement(node); + default: + return visitEachChild(node, visit, context); + } + } + function visitVariableStatement(node: VariableStatement) { + addSyntheticLeadingComment(node, SyntaxKind.SingleLineCommentTrivia, "@after"); + return node; + } + }; + + emitsCorrectly("before", sources, { before: [before] }); + emitsCorrectly("after", sources, { after: [after] }); + emitsCorrectly("both", sources, { before: [before], after: [after] }); + }); +} \ No newline at end of file diff --git a/src/harness/unittests/transform.ts b/src/harness/unittests/transform.ts new file mode 100644 index 00000000000..1a05138b84b --- /dev/null +++ b/src/harness/unittests/transform.ts @@ -0,0 +1,43 @@ +/// +/// + +namespace ts { + describe("TransformAPI", () => { + function transformsCorrectly(name: string, source: string, transformers: Transformer[]) { + it(name, () => { + Harness.Baseline.runBaseline(`transformApi/transformsCorrectly.${name}.js`, () => { + const transformed = transform(createSourceFile("source.ts", source, ScriptTarget.ES2015), transformers); + const printer = createPrinter({ newLine: NewLineKind.CarriageReturnLineFeed }, { + onEmitNode: transformed.emitNodeWithNotification, + onSubstituteNode: transformed.emitNodeWithSubstitution + }); + const result = printer.printBundle(createBundle(transformed.transformed)); + transformed.dispose(); + return result; + }); + }); + } + + transformsCorrectly("substitution", ` + var a = undefined; + `, [ + context => { + const previousOnSubstituteNode = context.onSubstituteNode; + context.enableSubstitution(SyntaxKind.Identifier); + context.onSubstituteNode = (hint, node) => { + node = previousOnSubstituteNode(hint, node); + if (hint === EmitHint.Expression && node.kind === SyntaxKind.Identifier && (node).text === "undefined") { + node = createPartiallyEmittedExpression( + addSyntheticTrailingComment( + setTextRange( + createVoidZero(), + node), + SyntaxKind.MultiLineCommentTrivia, "undefined")); + } + return node; + }; + return file => file; + } + ]); + }); +} diff --git a/src/services/outliningElementsCollector.ts b/src/services/outliningElementsCollector.ts index 2ad20a7ed0c..43054a10f5d 100644 --- a/src/services/outliningElementsCollector.ts +++ b/src/services/outliningElementsCollector.ts @@ -67,10 +67,10 @@ namespace ts.OutliningElementsCollector { // Only outline spans of two or more consecutive single line comments if (count > 1) { - const multipleSingleLineComments = { + const multipleSingleLineComments: CommentRange = { + kind: SyntaxKind.SingleLineCommentTrivia, pos: start, end: end, - kind: SyntaxKind.SingleLineCommentTrivia }; addOutliningSpanComments(multipleSingleLineComments, /*autoCollapse*/ false); diff --git a/src/services/services.ts b/src/services/services.ts index b3e3093cddb..359699e47ad 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -1447,7 +1447,8 @@ namespace ts { }); } - const emitOutput = program.emit(sourceFile, writeFile, cancellationToken, emitOnlyDtsFiles); + const customTransformers = host.getCustomTransformers && host.getCustomTransformers(); + const emitOutput = program.emit(sourceFile, writeFile, cancellationToken, emitOnlyDtsFiles, customTransformers); return { outputFiles, diff --git a/src/services/transform.ts b/src/services/transform.ts new file mode 100644 index 00000000000..724075e4cf4 --- /dev/null +++ b/src/services/transform.ts @@ -0,0 +1,34 @@ +/// +/// +namespace ts { + export interface TransformOptions { + newLine?: NewLineKind; + } + + /** + * Transform one or more source files using the supplied transformers. + * @param source A `SourceFile` or an array of `SourceFiles`. + * @param transformers An array of `Transformer` callbacks used to process the transformation. + * @param compilerOptions Optional compiler options. + */ + export function transform(source: SourceFile | SourceFile[], transformers: Transformer[], transformOptions?: TransformOptions) { + const compilerOptions = transformOptions || {}; + const newLine = getNewLineCharacter(compilerOptions); + const sourceFiles = isArray(source) ? source : [source]; + const fileMap = arrayToMap(sourceFiles, sourceFile => sourceFile.fileName); + const emitHost: EmitHost = { + getCompilerOptions: () => compilerOptions, + getCanonicalFileName: fileName => fileName, + getCommonSourceDirectory: () => "", + getCurrentDirectory: () => "", + getNewLine: () => newLine, + getSourceFile: fileName => fileMap.get(fileName), + getSourceFileByPath: fileName => fileMap.get(fileName), + getSourceFiles: () => sourceFiles, + isSourceFileFromExternalLibrary: () => false, + isEmitBlocked: () => false, + writeFile: () => Debug.fail("'writeFile()' is not supported during transformation.") + }; + return transformFiles(/*resolver*/ undefined, emitHost, sourceFiles, transformers); + } +} \ No newline at end of file diff --git a/src/services/transpile.ts b/src/services/transpile.ts index 0b90e9d030b..86c6a3e8904 100644 --- a/src/services/transpile.ts +++ b/src/services/transpile.ts @@ -123,7 +123,8 @@ let commandLineOptionsStringToEnum: CommandLineOptionOfCustomType[]; /** JS users may pass in string values for enum compiler options (such as ModuleKind), so convert. */ - function fixupCompilerOptions(options: CompilerOptions, diagnostics: Diagnostic[]): CompilerOptions { + /*@internal*/ + export function fixupCompilerOptions(options: CompilerOptions, diagnostics: Diagnostic[]): CompilerOptions { // Lazily create this value to fix module loading errors. commandLineOptionsStringToEnum = commandLineOptionsStringToEnum || filter(optionDeclarations, o => typeof o.type === "object" && !forEachEntry(o.type, v => typeof v !== "number")); diff --git a/src/services/tsconfig.json b/src/services/tsconfig.json index b4e8289f367..9bebdb6932e 100644 --- a/src/services/tsconfig.json +++ b/src/services/tsconfig.json @@ -57,6 +57,7 @@ "preProcess.ts", "rename.ts", "services.ts", + "transform.ts", "transpile.ts", "shims.ts", "signatureHelp.ts", diff --git a/src/services/types.ts b/src/services/types.ts index b4210538518..0b612d281a4 100644 --- a/src/services/types.ts +++ b/src/services/types.ts @@ -170,6 +170,11 @@ namespace ts { * completions will not be provided */ getDirectories?(directoryName: string): string[]; + + /** + * Gets a set of custom transformers to use during emit. + */ + getCustomTransformers?(): CustomTransformers | undefined; } // diff --git a/tests/baselines/reference/customTransforms/after.js b/tests/baselines/reference/customTransforms/after.js new file mode 100644 index 00000000000..63c95725f43 --- /dev/null +++ b/tests/baselines/reference/customTransforms/after.js @@ -0,0 +1,15 @@ +// [source.js] +function f1() { } +//@after +var c = (function () { + function c() { + } + return c; +}()); +(function () { }); +//@after +var e; +(function (e) { +})(e || (e = {})); +// leading +function f2() { } // trailing diff --git a/tests/baselines/reference/customTransforms/before.js b/tests/baselines/reference/customTransforms/before.js new file mode 100644 index 00000000000..4ee133afdbc --- /dev/null +++ b/tests/baselines/reference/customTransforms/before.js @@ -0,0 +1,15 @@ +// [source.js] +/*@before*/ +function f1() { } +var c = (function () { + function c() { + } + return c; +}()); +(function () { }); +var e; +(function (e) { +})(e || (e = {})); +// leading +/*@before*/ +function f2() { } // trailing diff --git a/tests/baselines/reference/customTransforms/both.js b/tests/baselines/reference/customTransforms/both.js new file mode 100644 index 00000000000..3013e7f8780 --- /dev/null +++ b/tests/baselines/reference/customTransforms/both.js @@ -0,0 +1,17 @@ +// [source.js] +/*@before*/ +function f1() { } +//@after +var c = (function () { + function c() { + } + return c; +}()); +(function () { }); +//@after +var e; +(function (e) { +})(e || (e = {})); +// leading +/*@before*/ +function f2() { } // trailing diff --git a/tests/baselines/reference/transformApi/transformsCorrectly.substitution.js b/tests/baselines/reference/transformApi/transformsCorrectly.substitution.js new file mode 100644 index 00000000000..40686768759 --- /dev/null +++ b/tests/baselines/reference/transformApi/transformsCorrectly.substitution.js @@ -0,0 +1 @@ +var a = void 0 /*undefined*/;