From b747c2dd96ede26c853dee1d8882d73bb6d8ddf8 Mon Sep 17 00:00:00 2001 From: Ron Buckton Date: Thu, 3 Aug 2017 18:30:09 -0700 Subject: [PATCH] exclude node_modules unless explicitly included --- src/compiler/commandLineParser.ts | 6 +- src/compiler/core.ts | 101 ++++++--- src/harness/unittests/matchFiles.ts | 193 ++++++++++++------ .../nodeModulesMaxDepthExceeded.errors.txt | 2 +- .../nodeModulesMaxDepthExceeded.errors.txt | 2 +- .../maxDepthExceeded/tsconfig.json | 2 +- 6 files changed, 200 insertions(+), 106 deletions(-) diff --git a/src/compiler/commandLineParser.ts b/src/compiler/commandLineParser.ts index 554b46dcdfe..b0cb7f94029 100644 --- a/src/compiler/commandLineParser.ts +++ b/src/compiler/commandLineParser.ts @@ -1446,14 +1446,10 @@ namespace ts { } } else { - // If no includes were specified, exclude common package folders and the outDir - const specs = includeSpecs ? [] : ["node_modules", "bower_components", "jspm_packages"]; - const outDir = raw["compilerOptions"] && raw["compilerOptions"]["outDir"]; if (outDir) { - specs.push(outDir); + excludeSpecs = [outDir]; } - excludeSpecs = specs; } if (fileNames === undefined && includeSpecs === undefined) { diff --git a/src/compiler/core.ts b/src/compiler/core.ts index 728fb433c04..07f979333b6 100644 --- a/src/compiler/core.ts +++ b/src/compiler/core.ts @@ -1889,14 +1889,54 @@ namespace ts { const reservedCharacterPattern = /[^\w\s\/]/g; const wildcardCharCodes = [CharacterCodes.asterisk, CharacterCodes.question]; - /** - * Matches any single directory segment unless it is the last segment and a .min.js file - * Breakdown: - * [^./] # matches everything up to the first . character (excluding directory seperators) - * (\\.(?!min\\.js$))? # matches . characters but not if they are part of the .min.js file extension - */ - const singleAsteriskRegexFragmentFiles = "([^./]|(\\.(?!min\\.js$))?)*"; - const singleAsteriskRegexFragmentOther = "[^/]*"; + /* @internal */ + export const commonPackageFolders: ReadonlyArray = ["node_modules", "bower_components", "jspm_packages"]; + + const implicitExcludePathRegexPattern = `(?!(${commonPackageFolders.join("|")})(/|$))`; + + interface WildcardMatcher { + singleAsteriskRegexFragment: string; + doubleAsteriskRegexFragment: string; + replaceWildcardCharacter: (match: string) => string; + } + + const filesMatcher: WildcardMatcher = { + /** + * Matches any single directory segment unless it is the last segment and a .min.js file + * Breakdown: + * [^./] # matches everything up to the first . character (excluding directory seperators) + * (\\.(?!min\\.js$))? # matches . characters but not if they are part of the .min.js file extension + */ + singleAsteriskRegexFragment: "([^./]|(\\.(?!min\\.js$))?)*", + /** + * Regex for the ** wildcard. Matches any number of subdirectories. When used for including + * files or directories, does not match subdirectories that start with a . character + */ + doubleAsteriskRegexFragment: `(/${implicitExcludePathRegexPattern}[^/.][^/]*)*?`, + replaceWildcardCharacter: match => replaceWildcardCharacter(match, filesMatcher.singleAsteriskRegexFragment) + }; + + const directoriesMatcher: WildcardMatcher = { + singleAsteriskRegexFragment: "[^/]*", + /** + * Regex for the ** wildcard. Matches any number of subdirectories. When used for including + * files or directories, does not match subdirectories that start with a . character + */ + doubleAsteriskRegexFragment: `(/${implicitExcludePathRegexPattern}[^/.][^/]*)*?`, + replaceWildcardCharacter: match => replaceWildcardCharacter(match, directoriesMatcher.singleAsteriskRegexFragment) + }; + + const excludeMatcher: WildcardMatcher = { + singleAsteriskRegexFragment: "[^/]*", + doubleAsteriskRegexFragment: "(/.+?)?", + replaceWildcardCharacter: match => replaceWildcardCharacter(match, excludeMatcher.singleAsteriskRegexFragment) + }; + + const wildcardMatchers = { + files: filesMatcher, + directories: directoriesMatcher, + exclude: excludeMatcher + }; export function getRegularExpressionForWildcard(specs: ReadonlyArray, basePath: string, usage: "files" | "directories" | "exclude"): string | undefined { const patterns = getRegularExpressionsForWildcards(specs, basePath, usage); @@ -1915,17 +1955,8 @@ namespace ts { return undefined; } - const replaceWildcardCharacter = usage === "files" ? replaceWildCardCharacterFiles : replaceWildCardCharacterOther; - const singleAsteriskRegexFragment = usage === "files" ? singleAsteriskRegexFragmentFiles : singleAsteriskRegexFragmentOther; - - /** - * Regex for the ** wildcard. Matches any number of subdirectories. When used for including - * files or directories, does not match subdirectories that start with a . character - */ - const doubleAsteriskRegexFragment = usage === "exclude" ? "(/.+?)?" : "(/[^/.][^/]*)*?"; - return flatMap(specs, spec => - spec && getSubPatternFromSpec(spec, basePath, usage, singleAsteriskRegexFragment, doubleAsteriskRegexFragment, replaceWildcardCharacter)); + spec && getSubPatternFromSpec(spec, basePath, usage, wildcardMatchers[usage])); } /** @@ -1936,7 +1967,7 @@ namespace ts { return !/[.*?]/.test(lastPathComponent); } - function getSubPatternFromSpec(spec: string, basePath: string, usage: "files" | "directories" | "exclude", singleAsteriskRegexFragment: string, doubleAsteriskRegexFragment: string, replaceWildcardCharacter: (match: string) => string): string | undefined { + function getSubPatternFromSpec(spec: string, basePath: string, usage: "files" | "directories" | "exclude", { singleAsteriskRegexFragment, doubleAsteriskRegexFragment, replaceWildcardCharacter }: WildcardMatcher): string | undefined { let subpattern = ""; let hasRecursiveDirectoryWildcard = false; let hasWrittenComponent = false; @@ -1975,20 +2006,36 @@ namespace ts { } if (usage !== "exclude") { + let componentPattern = ""; // The * and ? wildcards should not match directories or files that start with . if they // appear first in a component. Dotted directories and files can be included explicitly // like so: **/.*/.* if (component.charCodeAt(0) === CharacterCodes.asterisk) { - subpattern += "([^./]" + singleAsteriskRegexFragment + ")?"; + componentPattern += "([^./]" + singleAsteriskRegexFragment + ")?"; component = component.substr(1); } else if (component.charCodeAt(0) === CharacterCodes.question) { - subpattern += "[^./]"; + componentPattern += "[^./]"; component = component.substr(1); } - } - subpattern += component.replace(reservedCharacterPattern, replaceWildcardCharacter); + componentPattern += component.replace(reservedCharacterPattern, replaceWildcardCharacter); + + // Patterns should not include subfolders like node_modules unless they are + // explicitly included as part of the path. + // + // As an optimization, if the component pattern is the same as the component, + // then there definitely were no wildcard characters and we do not need to + // add the exclusion pattern. + if (componentPattern !== component) { + subpattern += implicitExcludePathRegexPattern; + } + + subpattern += componentPattern; + } + else { + subpattern += component.replace(reservedCharacterPattern, replaceWildcardCharacter); + } } hasWrittenComponent = true; @@ -2002,14 +2049,6 @@ namespace ts { return subpattern; } - function replaceWildCardCharacterFiles(match: string) { - return replaceWildcardCharacter(match, singleAsteriskRegexFragmentFiles); - } - - function replaceWildCardCharacterOther(match: string) { - return replaceWildcardCharacter(match, singleAsteriskRegexFragmentOther); - } - function replaceWildcardCharacter(match: string, singleAsteriskRegexFragment: string) { return match === "*" ? singleAsteriskRegexFragment : match === "?" ? "[^/]" : "\\" + match; } diff --git a/src/harness/unittests/matchFiles.ts b/src/harness/unittests/matchFiles.ts index e0454671930..71b1bfff11a 100644 --- a/src/harness/unittests/matchFiles.ts +++ b/src/harness/unittests/matchFiles.ts @@ -73,6 +73,7 @@ namespace ts { "c:/dev/a.d.ts", "c:/dev/a.js", "c:/dev/b.ts", + "c:/dev/x/a.ts", "c:/dev/node_modules/a.ts", "c:/dev/bower_components/a.ts", "c:/dev/jspm_packages/a.ts" @@ -141,7 +142,8 @@ namespace ts { errors: [], fileNames: [ "c:/dev/a.ts", - "c:/dev/b.ts" + "c:/dev/b.ts", + "c:/dev/x/a.ts" ], wildcardDirectories: { "c:/dev": ts.WatchDirectoryFlags.Recursive @@ -462,7 +464,6 @@ namespace ts { }; validateMatches(expected, json, caseInsensitiveHost, caseInsensitiveBasePath); }); - it("same named declarations are excluded", () => { const json = { include: [ @@ -651,71 +652,127 @@ namespace ts { }; validateMatches(expected, json, caseInsensitiveHost, caseInsensitiveBasePath); }); - it("with common package folders and no exclusions", () => { - const json = { - include: [ - "**/a.ts" - ] - }; - const expected: ts.ParsedCommandLine = { - options: {}, - errors: [], - fileNames: [ - "c:/dev/a.ts", - "c:/dev/bower_components/a.ts", - "c:/dev/jspm_packages/a.ts", - "c:/dev/node_modules/a.ts" - ], - wildcardDirectories: { - "c:/dev": ts.WatchDirectoryFlags.Recursive - }, - }; - validateMatches(expected, json, caseInsensitiveCommonFoldersHost, caseInsensitiveBasePath); - }); - it("with common package folders and exclusions", () => { - const json = { - include: [ - "**/a.ts" - ], - exclude: [ - "a.ts" - ] - }; - const expected: ts.ParsedCommandLine = { - options: {}, - errors: [], - fileNames: [ - "c:/dev/bower_components/a.ts", - "c:/dev/jspm_packages/a.ts", - "c:/dev/node_modules/a.ts" - ], - wildcardDirectories: { - "c:/dev": ts.WatchDirectoryFlags.Recursive - }, - }; - validateMatches(expected, json, caseInsensitiveCommonFoldersHost, caseInsensitiveBasePath); - }); - it("with common package folders and empty exclude", () => { - const json = { - include: [ - "**/a.ts" - ], - exclude: [] - }; - const expected: ts.ParsedCommandLine = { - options: {}, - errors: [], - fileNames: [ - "c:/dev/a.ts", - "c:/dev/bower_components/a.ts", - "c:/dev/jspm_packages/a.ts", - "c:/dev/node_modules/a.ts" - ], - wildcardDirectories: { - "c:/dev": ts.WatchDirectoryFlags.Recursive - }, - }; - validateMatches(expected, json, caseInsensitiveCommonFoldersHost, caseInsensitiveBasePath); + describe("with common package folders", () => { + it("and no exclusions", () => { + const json = { + include: [ + "**/a.ts" + ] + }; + const expected: ts.ParsedCommandLine = { + options: {}, + errors: [], + fileNames: [ + "c:/dev/a.ts", + "c:/dev/x/a.ts" + ], + wildcardDirectories: { + "c:/dev": ts.WatchDirectoryFlags.Recursive + }, + }; + validateMatches(expected, json, caseInsensitiveCommonFoldersHost, caseInsensitiveBasePath); + }); + it("and exclusions", () => { + const json = { + include: [ + "**/?.ts" + ], + exclude: [ + "a.ts" + ] + }; + const expected: ts.ParsedCommandLine = { + options: {}, + errors: [], + fileNames: [ + "c:/dev/b.ts", + "c:/dev/x/a.ts" + ], + wildcardDirectories: { + "c:/dev": ts.WatchDirectoryFlags.Recursive + }, + }; + validateMatches(expected, json, caseInsensitiveCommonFoldersHost, caseInsensitiveBasePath); + }); + it("and empty exclude", () => { + const json = { + include: [ + "**/a.ts" + ], + exclude: [] + }; + const expected: ts.ParsedCommandLine = { + options: {}, + errors: [], + fileNames: [ + "c:/dev/a.ts", + "c:/dev/x/a.ts" + ], + wildcardDirectories: { + "c:/dev": ts.WatchDirectoryFlags.Recursive + }, + }; + validateMatches(expected, json, caseInsensitiveCommonFoldersHost, caseInsensitiveBasePath); + }); + it("and explicit recursive include", () => { + const json = { + include: [ + "**/a.ts", + "**/node_modules/a.ts" + ] + }; + const expected: ts.ParsedCommandLine = { + options: {}, + errors: [], + fileNames: [ + "c:/dev/a.ts", + "c:/dev/x/a.ts", + "c:/dev/node_modules/a.ts" + ], + wildcardDirectories: { + "c:/dev": ts.WatchDirectoryFlags.Recursive + }, + }; + validateMatches(expected, json, caseInsensitiveCommonFoldersHost, caseInsensitiveBasePath); + }); + it("and wildcard include", () => { + const json = { + include: [ + "*/a.ts" + ] + }; + const expected: ts.ParsedCommandLine = { + options: {}, + errors: [], + fileNames: [ + "c:/dev/x/a.ts" + ], + wildcardDirectories: { + "c:/dev": ts.WatchDirectoryFlags.Recursive + }, + }; + validateMatches(expected, json, caseInsensitiveCommonFoldersHost, caseInsensitiveBasePath); + }); + it("and explicit wildcard include", () => { + const json = { + include: [ + "*/a.ts", + "node_modules/a.ts" + ] + }; + const expected: ts.ParsedCommandLine = { + options: {}, + errors: [], + fileNames: [ + "c:/dev/x/a.ts", + "c:/dev/node_modules/a.ts" + ], + wildcardDirectories: { + "c:/dev": ts.WatchDirectoryFlags.Recursive + }, + }; + validateMatches(expected, json, caseInsensitiveCommonFoldersHost, caseInsensitiveBasePath); + }); }); it("exclude .js files when allowJs=false", () => { const json = { @@ -1066,6 +1123,7 @@ namespace ts { }; validateMatches(expected, json, caseInsensitiveHost, caseInsensitiveBasePath); }); + describe("with trailing recursive directory", () => { it("in includes", () => { const json = { @@ -1264,6 +1322,7 @@ namespace ts { }); }); }); + describe("with files or folders that begin with a .", () => { it("that are not explicitly included", () => { const json = { diff --git a/tests/baselines/reference/project/nodeModulesMaxDepthExceeded/amd/nodeModulesMaxDepthExceeded.errors.txt b/tests/baselines/reference/project/nodeModulesMaxDepthExceeded/amd/nodeModulesMaxDepthExceeded.errors.txt index a1b170ce665..bd2dd6231f3 100644 --- a/tests/baselines/reference/project/nodeModulesMaxDepthExceeded/amd/nodeModulesMaxDepthExceeded.errors.txt +++ b/tests/baselines/reference/project/nodeModulesMaxDepthExceeded/amd/nodeModulesMaxDepthExceeded.errors.txt @@ -9,7 +9,7 @@ maxDepthExceeded/root.ts(4,4): error TS2540: Cannot assign to 'rel' because it i "maxNodeModuleJsDepth": 1, // Note: Module m1 is already included as a root file "outDir": "built" }, - "include": ["**/*"], + "include": ["**/*", "node_modules/**/*"], "exclude": ["node_modules/m2/**/*"] } diff --git a/tests/baselines/reference/project/nodeModulesMaxDepthExceeded/node/nodeModulesMaxDepthExceeded.errors.txt b/tests/baselines/reference/project/nodeModulesMaxDepthExceeded/node/nodeModulesMaxDepthExceeded.errors.txt index a1b170ce665..bd2dd6231f3 100644 --- a/tests/baselines/reference/project/nodeModulesMaxDepthExceeded/node/nodeModulesMaxDepthExceeded.errors.txt +++ b/tests/baselines/reference/project/nodeModulesMaxDepthExceeded/node/nodeModulesMaxDepthExceeded.errors.txt @@ -9,7 +9,7 @@ maxDepthExceeded/root.ts(4,4): error TS2540: Cannot assign to 'rel' because it i "maxNodeModuleJsDepth": 1, // Note: Module m1 is already included as a root file "outDir": "built" }, - "include": ["**/*"], + "include": ["**/*", "node_modules/**/*"], "exclude": ["node_modules/m2/**/*"] } diff --git a/tests/cases/projects/NodeModulesSearch/maxDepthExceeded/tsconfig.json b/tests/cases/projects/NodeModulesSearch/maxDepthExceeded/tsconfig.json index 52633bb5a98..b2ee28482ba 100644 --- a/tests/cases/projects/NodeModulesSearch/maxDepthExceeded/tsconfig.json +++ b/tests/cases/projects/NodeModulesSearch/maxDepthExceeded/tsconfig.json @@ -4,6 +4,6 @@ "maxNodeModuleJsDepth": 1, // Note: Module m1 is already included as a root file "outDir": "built" }, - "include": ["**/*"], + "include": ["**/*", "node_modules/**/*"], "exclude": ["node_modules/m2/**/*"] }