From 2b2092b1a22e6c3e8c341b56b670c3ad7c5e4d59 Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Mon, 14 Mar 2016 13:30:38 -0700 Subject: [PATCH] find module augmentations in preprocessor --- src/services/services.ts | 144 ++++++++---- .../unittests/services/preProcessFile.ts | 205 ++++++++++++++++-- 2 files changed, 282 insertions(+), 67 deletions(-) diff --git a/src/services/services.ts b/src/services/services.ts index a562ecf18cf..1c079b84027 100644 --- a/src/services/services.ts +++ b/src/services/services.ts @@ -2147,8 +2147,23 @@ namespace ts { export function preProcessFile(sourceText: string, readImportFiles = true, detectJavaScriptImports = false): PreProcessedFileInfo { const referencedFiles: FileReference[] = []; const importedFiles: FileReference[] = []; - let ambientExternalModules: string[]; + let ambientExternalModules: { ref: FileReference, depth: number }[]; let isNoDefaultLib = false; + let braceNesting = 0; + // assume that text represent an external module if it contains at least one top level import/export + // ambient modules that are found inside external modules are interpreted as module augmentations + let externalModule = false; + + function nextToken() { + const token = scanner.scan(); + if (token === SyntaxKind.OpenBraceToken) { + braceNesting++; + } + else if (token === SyntaxKind.CloseBraceToken) { + braceNesting--; + } + return token; + } function processTripleSlashDirectives(): void { const commentRanges = getLeadingCommentRanges(sourceText, 0); @@ -2165,21 +2180,33 @@ namespace ts { }); } + function getFileReference() { + const file = scanner.getTokenValue(); + const pos = scanner.getTokenPos(); + return { + fileName: file, + pos: pos, + end: pos + file.length + }; + } + function recordAmbientExternalModule(): void { if (!ambientExternalModules) { ambientExternalModules = []; } - ambientExternalModules.push(scanner.getTokenValue()); + ambientExternalModules.push({ ref: getFileReference(), depth: braceNesting }); } function recordModuleName() { - const importPath = scanner.getTokenValue(); - const pos = scanner.getTokenPos(); - importedFiles.push({ - fileName: importPath, - pos: pos, - end: pos + importPath.length - }); + importedFiles.push(getFileReference()); + + markAsExternalModuleIfTopLevel(); + } + + function markAsExternalModuleIfTopLevel() { + if (braceNesting === 0) { + externalModule = true; + } } /** @@ -2189,9 +2216,9 @@ namespace ts { let token = scanner.getToken(); if (token === SyntaxKind.DeclareKeyword) { // declare module "mod" - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.ModuleKeyword) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.StringLiteral) { recordAmbientExternalModule(); } @@ -2208,7 +2235,8 @@ namespace ts { function tryConsumeImport(): boolean { let token = scanner.getToken(); if (token === SyntaxKind.ImportKeyword) { - token = scanner.scan(); + + token = nextToken(); if (token === SyntaxKind.StringLiteral) { // import "mod"; recordModuleName(); @@ -2216,9 +2244,9 @@ namespace ts { } else { if (token === SyntaxKind.Identifier || isKeyword(token)) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.FromKeyword) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.StringLiteral) { // import d from "mod"; recordModuleName(); @@ -2232,7 +2260,7 @@ namespace ts { } else if (token === SyntaxKind.CommaToken) { // consume comma and keep going - token = scanner.scan(); + token = nextToken(); } else { // unknown syntax @@ -2241,17 +2269,17 @@ namespace ts { } if (token === SyntaxKind.OpenBraceToken) { - token = scanner.scan(); + token = nextToken(); // consume "{ a as B, c, d as D}" clauses // make sure that it stops on EOF while (token !== SyntaxKind.CloseBraceToken && token !== SyntaxKind.EndOfFileToken) { - token = scanner.scan(); + token = nextToken(); } if (token === SyntaxKind.CloseBraceToken) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.FromKeyword) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.StringLiteral) { // import {a as A} from "mod"; // import d, {a, b as B} from "mod" @@ -2261,13 +2289,13 @@ namespace ts { } } else if (token === SyntaxKind.AsteriskToken) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.AsKeyword) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.Identifier || isKeyword(token)) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.FromKeyword) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.StringLiteral) { // import * as NS from "mod" // import d, * as NS from "mod" @@ -2288,19 +2316,20 @@ namespace ts { function tryConsumeExport(): boolean { let token = scanner.getToken(); if (token === SyntaxKind.ExportKeyword) { - token = scanner.scan(); + markAsExternalModuleIfTopLevel(); + token = nextToken(); if (token === SyntaxKind.OpenBraceToken) { - token = scanner.scan(); + token = nextToken(); // consume "{ a as B, c, d as D}" clauses // make sure it stops on EOF while (token !== SyntaxKind.CloseBraceToken && token !== SyntaxKind.EndOfFileToken) { - token = scanner.scan(); + token = nextToken(); } if (token === SyntaxKind.CloseBraceToken) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.FromKeyword) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.StringLiteral) { // export {a as A} from "mod"; // export {a, b as B} from "mod" @@ -2310,9 +2339,9 @@ namespace ts { } } else if (token === SyntaxKind.AsteriskToken) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.FromKeyword) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.StringLiteral) { // export * from "mod" recordModuleName(); @@ -2320,9 +2349,9 @@ namespace ts { } } else if (token === SyntaxKind.ImportKeyword) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.Identifier || isKeyword(token)) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.EqualsToken) { if (tryConsumeRequireCall(/*skipCurrentToken*/ true)) { return true; @@ -2338,11 +2367,11 @@ namespace ts { } function tryConsumeRequireCall(skipCurrentToken: boolean): boolean { - let token = skipCurrentToken ? scanner.scan() : scanner.getToken(); + let token = skipCurrentToken ? nextToken() : scanner.getToken(); if (token === SyntaxKind.RequireKeyword) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.OpenParenToken) { - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.StringLiteral) { // require("mod"); recordModuleName(); @@ -2356,17 +2385,17 @@ namespace ts { function tryConsumeDefine(): boolean { let token = scanner.getToken(); if (token === SyntaxKind.Identifier && scanner.getTokenValue() === "define") { - token = scanner.scan(); + token = nextToken(); if (token !== SyntaxKind.OpenParenToken) { return true; } - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.StringLiteral) { // looks like define ("modname", ... - skip string literal and comma - token = scanner.scan(); + token = nextToken(); if (token === SyntaxKind.CommaToken) { - token = scanner.scan(); + token = nextToken(); } else { // unexpected token @@ -2380,7 +2409,7 @@ namespace ts { } // skip open bracket - token = scanner.scan(); + token = nextToken(); let i = 0; // scan until ']' or EOF while (token !== SyntaxKind.CloseBracketToken && token !== SyntaxKind.EndOfFileToken) { @@ -2390,7 +2419,7 @@ namespace ts { i++; } - token = scanner.scan(); + token = nextToken(); } return true; @@ -2400,7 +2429,7 @@ namespace ts { function processImports(): void { scanner.setText(sourceText); - scanner.scan(); + nextToken(); // Look for: // import "mod"; // import d from "mod" @@ -2427,7 +2456,7 @@ namespace ts { continue; } else { - scanner.scan(); + nextToken(); } } @@ -2438,7 +2467,34 @@ namespace ts { processImports(); } processTripleSlashDirectives(); - return { referencedFiles, importedFiles, isLibFile: isNoDefaultLib, ambientExternalModules }; + if (externalModule) { + // for external modules module all nested ambient modules are augmentations + if (ambientExternalModules) { + // move all detected ambient modules to imported files since they need to be resolved + for (const decl of ambientExternalModules) { + importedFiles.push(decl.ref); + } + } + return { referencedFiles, importedFiles, isLibFile: isNoDefaultLib, ambientExternalModules: undefined }; + } + else { + // for global scripts ambient modules still can have augmentations - look for ambient modules with depth > 0 + let ambientModuleNames: string[]; + if (ambientExternalModules) { + for (const decl of ambientExternalModules) { + if (decl.depth === 0) { + if (!ambientModuleNames) { + ambientModuleNames = []; + } + ambientModuleNames.push(decl.ref.fileName); + } + else { + importedFiles.push(decl.ref); + } + } + } + return { referencedFiles, importedFiles, isLibFile: isNoDefaultLib, ambientExternalModules: ambientModuleNames }; + } } /// Helpers diff --git a/tests/cases/unittests/services/preProcessFile.ts b/tests/cases/unittests/services/preProcessFile.ts index d9ddaf0f256..a648a3c4b26 100644 --- a/tests/cases/unittests/services/preProcessFile.ts +++ b/tests/cases/unittests/services/preProcessFile.ts @@ -1,6 +1,10 @@ /// /// +declare namespace chai.assert { + function deepEqual(actual: any, expected: any): void; +} + describe('PreProcessFile:', function () { function test(sourceText: string, readImportFile: boolean, detectJavaScriptImports: boolean, expectedPreProcess: ts.PreProcessedFileInfo): void { var resultPreProcess = ts.preProcessFile(sourceText, readImportFile, detectJavaScriptImports); @@ -15,34 +19,30 @@ describe('PreProcessFile:', function () { assert.equal(resultIsLibFile, expectedIsLibFile, "Pre-processed file has different value for isLibFile. Expected: " + expectedPreProcess + ". Actual: " + resultIsLibFile); - assert.equal(resultImportedFiles.length, expectedImportedFiles.length, - "Array's length of imported files does not match expected. Expected: " + expectedImportedFiles.length + ". Actual: " + resultImportedFiles.length); + checkFileReferenceList("Imported files", expectedImportedFiles, resultImportedFiles); + checkFileReferenceList("Referenced files", expectedReferencedFiles, resultReferencedFiles); - assert.equal(resultReferencedFiles.length, expectedReferencedFiles.length, - "Array's length of referenced files does not match expected. Expected: " + expectedReferencedFiles.length + ". Actual: " + resultReferencedFiles.length); + assert.deepEqual(resultPreProcess.ambientExternalModules, expectedPreProcess.ambientExternalModules); + } - for (var i = 0; i < expectedImportedFiles.length; ++i) { - var resultImportedFile = resultImportedFiles[i]; - var expectedImportedFile = expectedImportedFiles[i]; - - assert.equal(resultImportedFile.fileName, expectedImportedFile.fileName, "Imported file path does not match expected. Expected: " + expectedImportedFile.fileName + ". Actual: " + resultImportedFile.fileName + "."); - - assert.equal(resultImportedFile.pos, expectedImportedFile.pos, "Imported file position does not match expected. Expected: " + expectedImportedFile.pos + ". Actual: " + resultImportedFile.pos + "."); - - assert.equal(resultImportedFile.end, expectedImportedFile.end, "Imported file length does not match expected. Expected: " + expectedImportedFile.end + ". Actual: " + resultImportedFile.end + "."); + function checkFileReferenceList(kind: string, expected: ts.FileReference[], actual: ts.FileReference[]) { + if (expected === actual) { + return; } + if (!expected) { + assert.isTrue(false, `Expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); + } + assert.equal(actual.length, expected.length, `[${kind}] Actual array's length does not match expected length. Expected files: ${JSON.stringify(expected)}, actual files: ${JSON.stringify(actual)}`); - for (var i = 0; i < expectedReferencedFiles.length; ++i) { - var resultReferencedFile = resultReferencedFiles[i]; - var expectedReferencedFile = expectedReferencedFiles[i]; - - assert.equal(resultReferencedFile.fileName, expectedReferencedFile.fileName, "Referenced file path does not match expected. Expected: " + expectedReferencedFile.fileName + ". Actual: " + resultReferencedFile.fileName + "."); - - assert.equal(resultReferencedFile.pos, expectedReferencedFile.pos, "Referenced file position does not match expected. Expected: " + expectedReferencedFile.pos + ". Actual: " + resultReferencedFile.pos + "."); - - assert.equal(resultReferencedFile.end, expectedReferencedFile.end, "Referenced file length does not match expected. Expected: " + expectedReferencedFile.end + ". Actual: " + resultReferencedFile.end + "."); + for (var i = 0; i < expected.length; ++i) { + var actualReference = actual[i]; + var expectedReference = expected[i]; + assert.equal(actualReference.fileName, expectedReference.fileName, `[${kind}] actual file path does not match expected. Expected: "${expectedReference.fileName}". Actual: "${actualReference.fileName}".`); + assert.equal(actualReference.pos, expectedReference.pos, `[${kind}] actual file start position does not match expected. Expected: "${expectedReference.pos}". Actual: "${actualReference.pos}".`); + assert.equal(actualReference.end, expectedReference.end, `[${kind}] actual file end pos does not match expected. Expected: "${expectedReference.end}". Actual: "${actualReference.end}".`); } } + describe("Test preProcessFiles,", function () { it("Correctly return referenced files from triple slash", function () { test("///" + "\n" + "///" + "\n" + "///" + "\n" + "///", @@ -183,7 +183,7 @@ describe('PreProcessFile:', function () { function foo() { } `, - /* readImports */ false, + /* readImports */ true, /* detectJavaScriptImports */ false, { @@ -262,6 +262,165 @@ describe('PreProcessFile:', function () { isLibFile: false }) }); + it("correctly handles augmentations in external modules - 1", () => { + test(` + declare module "../Observable" { + interface I {} + } + + export {} + `, + /*readImportFile*/ true, + /*detectJavaScriptImports*/ false, + { + referencedFiles: [], + importedFiles: [ + { "fileName": "../Observable", "pos": 28, "end": 41 } + ], + ambientExternalModules: undefined, + isLibFile: false + }) + }); + it("correctly handles augmentations in external modules - 2", () => { + test(` + declare module "../Observable" { + interface I {} + } + + import * as x from "m"; + `, + /*readImportFile*/ true, + /*detectJavaScriptImports*/ false, + { + referencedFiles: [], + importedFiles: [ + { "fileName": "m", "pos": 135, "end": 136 }, + { "fileName": "../Observable", "pos": 28, "end": 41 } + ], + ambientExternalModules: undefined, + isLibFile: false + }) + }); + it("correctly handles augmentations in external modules - 3", () => { + test(` + declare module "../Observable" { + interface I {} + } + + import m = require("m"); + `, + /*readImportFile*/ true, + /*detectJavaScriptImports*/ false, + { + referencedFiles: [], + importedFiles: [ + { "fileName": "m", "pos": 135, "end": 136 }, + { "fileName": "../Observable", "pos": 28, "end": 41 } + ], + ambientExternalModules: undefined, + isLibFile: false + }) + }); + it("correctly handles augmentations in external modules - 4", () => { + test(` + declare module "../Observable" { + interface I {} + } + namespace N {} + export = N; + `, + /*readImportFile*/ true, + /*detectJavaScriptImports*/ false, + { + referencedFiles: [], + importedFiles: [ + { "fileName": "../Observable", "pos": 28, "end": 41 } + ], + ambientExternalModules: undefined, + isLibFile: false + }) + }); + it("correctly handles augmentations in external modules - 5", () => { + test(` + declare module "../Observable" { + interface I {} + } + namespace N {} + export import IN = N; + `, + /*readImportFile*/ true, + /*detectJavaScriptImports*/ false, + { + referencedFiles: [], + importedFiles: [ + { "fileName": "../Observable", "pos": 28, "end": 41 } + ], + ambientExternalModules: undefined, + isLibFile: false + }) + }); + it("correctly handles augmentations in external modules - 6", () => { + test(` + declare module "../Observable" { + interface I {} + } + export let x = 1; + `, + /*readImportFile*/ true, + /*detectJavaScriptImports*/ false, + { + referencedFiles: [], + importedFiles: [ + { "fileName": "../Observable", "pos": 28, "end": 41 } + ], + ambientExternalModules: undefined, + isLibFile: false + }) + }); + it ("correctly handles augmentations in ambient external modules - 1", () => { + test(` + declare module "m1" { + export * from "m2"; + declare module "augmentation" { + interface I {} + } + } + `, + /*readImportFile*/ true, + /*detectJavaScriptImports*/ false, + { + referencedFiles: [], + importedFiles: [ + { "fileName": "m2", "pos": 65, "end": 67 }, + { "fileName": "augmentation", "pos": 102, "end": 114 } + ], + ambientExternalModules: ["m1"], + isLibFile: false + }); + }); + it ("correctly handles augmentations in ambient external modules - 2", () => { + test(` + namespace M { var x; } + import IM = M; + declare module "m1" { + export * from "m2"; + declare module "augmentation" { + interface I {} + } + } + `, + /*readImportFile*/ true, + /*detectJavaScriptImports*/ false, + { + referencedFiles: [], + importedFiles: [ + { "fileName": "m2", "pos": 127, "end": 129 }, + { "fileName": "augmentation", "pos": 164, "end": 176 } + ], + ambientExternalModules: ["m1"], + isLibFile: false + }); + }); }); });