From d8572508ee0650e58a7602098498b1309c301d2d Mon Sep 17 00:00:00 2001 From: Ron Buckton Date: Mon, 14 Dec 2015 15:21:12 -0800 Subject: [PATCH] Heavily revised implementation that relies on an updated 'readDirectory' API. --- Jakefile.js | 1 + src/compiler/commandLineParser.ts | 753 +++--------------- src/compiler/core.ts | 203 ++++- src/compiler/sys.ts | 121 +-- src/compiler/types.ts | 22 +- src/harness/harness.ts | 39 +- src/harness/harnessLanguageService.ts | 10 +- src/harness/loggedIO.ts | 9 +- src/harness/projectsRunner.ts | 15 +- src/harness/rwcRunner.ts | 3 - src/harness/vfs.ts | 160 ++++ src/services/shims.ts | 76 +- .../cases/unittests/cachingInServerLSHost.ts | 4 +- tests/cases/unittests/expandFiles.ts | 259 +++--- tests/cases/unittests/session.ts | 2 - 15 files changed, 641 insertions(+), 1036 deletions(-) create mode 100644 src/harness/vfs.ts diff --git a/Jakefile.js b/Jakefile.js index 58677e19a75..0ce1d5a95ec 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -131,6 +131,7 @@ var languageServiceLibrarySources = [ var harnessCoreSources = [ "harness.ts", + "vfs.ts", "sourceMapRecorder.ts", "harnessLanguageService.ts", "fourslash.ts", diff --git a/src/compiler/commandLineParser.ts b/src/compiler/commandLineParser.ts index b4339e78f2a..c81b3b63244 100644 --- a/src/compiler/commandLineParser.ts +++ b/src/compiler/commandLineParser.ts @@ -582,47 +582,8 @@ namespace ts { return { options, errors }; } - // Simplified whitelist, forces escaping of any non-word (or digit), non-whitespace character. - const reservedCharacterPattern = /[^\w\s]/g; - - const enum ExpansionState { - Ok, - Error - } - - interface ExpansionContext { - /** A pattern used to exclude a file specification. */ - excludePattern: RegExp; - /** Compiler options. */ - options: CompilerOptions; - /** The host used to resolve files and directories. */ - host: ParseConfigHost; - /** Errors to report. */ - errors: Diagnostic[]; - /** The set of literal files. */ - literalFiles: FileMap; - /** The set of files matching a wildcard. */ - wildcardFiles: FileMap; - /** Directories to be watched. */ - wildcardDirectories: FileMap; - /** Supported extensions. */ - supportedExtensions: string[]; - /** - * Path cache, used to reduce calls to the file system. `true` indicates a file exists, - * `false` indicates a file or directory does not exist. A DirectoryResult - * indicates the file and subdirectory names in a directory. */ - cache: FileMap; - } - - const enum FileSystemEntryKind { - File, - Directory - } - - interface DirectoryResult { - files?: string[]; - directories?: string[]; - } + const invalidTrailingRecursionPattern = /(^|\/)\*\*\/?$/; + const invalidMultipleRecursionPatterns = /(^|\/)\*\*\/(.*\/)?\*\*($|\/)/; /** * Expands an array of file specifications. @@ -642,348 +603,143 @@ namespace ts { // The exclude spec list is converted into a regular expression, which allows us to quickly // test whether a file or directory should be excluded before recursively traversing the // file system. - const excludePattern = includeSpecs ? createExcludeRegularExpression(excludeSpecs, basePath, options, host, errors) : undefined; const keyMapper = host.useCaseSensitiveFileNames ? caseSensitiveKeyMapper : caseInsensitiveKeyMapper; // Literal file names (provided via the "files" array in tsconfig.json) are stored in a // file map with a possibly case insensitive key. We use this map later when when including // wildcard paths. - const literalFiles = createFileMap(keyMapper); + const literalFileMap: Map = {}; // Wildcard paths (provided via the "includes" array in tsconfig.json) are stored in a // file map with a possibly case insensitive key. We use this map to store paths matched // via wildcard, and to handle extension priority. - const wildcardFiles = createFileMap(keyMapper); + const wildcardFileMap: Map = {}; // Wildcard directories (provided as part of a wildcard path) are stored in a // file map that marks whether it was a regular wildcard match (with a `*` or `?` token), // or a recursive directory. This information is used by filesystem watchers to monitor for // new entries in these paths. - const wildcardDirectories = createFileMap(keyMapper); - - // To reduce the overhead of disk I/O (and marshalling to managed code when hosted in - // Visual Studio), file system queries are cached during the expansion session. - // If present, a cache entry can be one of three values: - // - A `false` value indicates the file or directory did not exist. - // - A `true` value indicates the path is a file and it exists. - // - An object value indicates the path is a directory and exists. The object may have - // zero, one, or both of the following properties: - // - A "files" array, which contains the file names in the directory. - // - A "directories" array, which contains the subdirectory names in the directory. - const cache = createFileMap(keyMapper); + const wildcardDirectories: Map = getWildcardDirectories(includeSpecs, basePath, host.useCaseSensitiveFileNames); // Rather than requery this for each file and filespec, we query the supported extensions // once and store it on the expansion context. const supportedExtensions = getSupportedExtensions(options); - // The expansion context holds references to shared information for the various expansion - // operations to reduce the overhead of closures. - const context: ExpansionContext = { - options, - host, - errors, - excludePattern, - literalFiles, - wildcardFiles, - wildcardDirectories, - supportedExtensions, - cache - }; - // Literal files are always included verbatim. An "include" or "exclude" specification cannot // remove a literal file. if (fileNames) { for (const fileName of fileNames) { - const path = toPath(fileName, basePath, caseSensitiveKeyMapper); - if (!literalFiles.contains(path)) { - literalFiles.set(path, path); + const file = combinePaths(basePath, fileName); + literalFileMap[keyMapper(file)] = file; + } + } + + if (includeSpecs) { + includeSpecs = validateSpecs(includeSpecs, errors, /*allowTrailingRecursion*/ false); + if (excludeSpecs) { + excludeSpecs = validateSpecs(excludeSpecs, errors, /*allowTrailingRecursion*/ true); + } + + for (const file of host.readDirectory(basePath, supportedExtensions, excludeSpecs, includeSpecs)) { + // If we have already included a literal or wildcard path with a + // higher priority extension, we should skip this file. + // + // This handles cases where we may encounter both .ts and + // .d.ts (or .js if "allowJs" is enabled) in the same + // directory when they are compilation outputs. + if (hasFileWithHigherPriorityExtension(file, literalFileMap, wildcardFileMap, supportedExtensions, keyMapper)) { + continue; + } + + // We may have included a wildcard path with a lower priority + // extension due to the user-defined order of entries in the + // "include" array. If there is a lower priority extension in the + // same directory, we should remove it. + removeWildcardFilesWithLowerPriorityExtension(file, wildcardFileMap, supportedExtensions, keyMapper); + + const key = keyMapper(file); + if (!hasProperty(literalFileMap, key) && !hasProperty(wildcardFileMap, key)) { + wildcardFileMap[key] = file; } } } - // Each "include" specification is expanded and matching files are added. - if (includeSpecs) { - for (let includeSpec of includeSpecs) { - includeSpec = normalizePath(includeSpec); - includeSpec = removeTrailingDirectorySeparator(includeSpec); - expandFileSpec(includeSpec, basePath, 0, context); - } - } - + const literalFiles = reduceProperties(literalFileMap, addFileToOutput, []); + const wildcardFiles = reduceProperties(wildcardFileMap, addFileToOutput, []); + wildcardFiles.sort(host.useCaseSensitiveFileNames ? compareStrings : compareStringsCaseInsensitive); return { - fileNames: wildcardFiles.reduceProperties(addFileToOutput, literalFiles.reduceProperties(addFileToOutput, [])), - wildcardDirectories: wildcardDirectories.reduceProperties>(addDirectoryToOutput, {}), + fileNames: literalFiles.concat(wildcardFiles), + wildcardDirectories }; } + function validateSpecs(specs: string[], errors: Diagnostic[], allowTrailingRecursion: boolean) { + const validSpecs: string[] = []; + for (const spec of specs) { + if (!allowTrailingRecursion && invalidTrailingRecursionPattern.test(spec)) { + errors.push(createCompilerDiagnostic(Diagnostics.File_specification_cannot_contain_multiple_recursive_directory_wildcards_Asterisk_Asterisk_Colon_0, spec)); + } + else if (invalidMultipleRecursionPatterns.test(spec)) { + errors.push(createCompilerDiagnostic(Diagnostics.File_specification_cannot_end_in_a_recursive_directory_wildcard_Asterisk_Asterisk_Colon_0, spec)); + } + else { + validSpecs.push(spec); + } + } + + return validSpecs; + } + + const watchRecursivePattern = /\/[^/]*?[*?][^/]*\//; + const wildcardDirectoryPattern = /^[^*?]*(?=\/[^/]*[*?])/; + /** - * Expands a file specification with wildcards. - * - * @param fileSpec The original file specification. - * @param basePath The directory to expand. This path must exist. - * @param start The starting offset in the file specification. - * @param context The expansion context. - * @param isExpandingRecursiveDirectory A value indicating whether the file specification includes a recursive directory wildcard prior to the start of this segment. + * Gets directories in a set of include patterns that should be watched for changes. */ - function expandFileSpec(fileSpec: string, basePath: Path, start: number, context: ExpansionContext, isExpandingRecursiveDirectory?: boolean): ExpansionState { - // A file specification must always point to a file. As a result, we always assume the - // path segment following the last directory separator points to a file. The only - // exception is when the final path segment is the recursive directory pattern "**", in - // which case we report an error. - const { host, options, errors, wildcardFiles, wildcardDirectories, excludePattern, cache, supportedExtensions } = context; - - // Skip expansion if the base path matches an exclude pattern. - if (isExcludedPath(basePath, excludePattern)) { - return ExpansionState.Ok; - } - - // Find the offset of the next wildcard in the file specification. If there are no more - // wildcards, we can include the file if it exists and isn't excluded. - let offset = indexOfWildcard(fileSpec, start); - if (offset < 0) { - const path = toPath(fileSpec.substring(start), basePath, caseSensitiveKeyMapper); - if (!isExcludedPath(path, excludePattern) && pathExists(path, FileSystemEntryKind.File, context)) { - includeFile(path, context, /*wildcardHasExtension*/ true); - } - - return ExpansionState.Ok; - } - - // Find the last directory separator before the wildcard to get the leading path. - offset = fileSpec.lastIndexOf(directorySeparator, offset); - if (offset > start) { - // The wildcard occurs in a later segment, include remaining path up to - // wildcard in prefix. - basePath = toPath(fileSpec.substring(start, offset), basePath, caseSensitiveKeyMapper); - - // Skip this wildcard path if the base path now matches an exclude pattern. - if (isExcludedPath(basePath, excludePattern) || !pathExists(basePath, FileSystemEntryKind.Directory, context)) { - return ExpansionState.Ok; - } - - start = offset + 1; - } - - // Find the offset of the next directory separator to extract the wildcard path segment. - offset = getEndOfPathSegment(fileSpec, start); - - // Check if the current offset is the beginning of a recursive directory pattern. - if (isRecursiveDirectoryWildcard(fileSpec, start, offset)) { - // Stop expansion if a file specification contains more than one recursive directory pattern. - if (isExpandingRecursiveDirectory) { - if (errors) { - errors.push(createCompilerDiagnostic(Diagnostics.File_specification_cannot_contain_multiple_recursive_directory_wildcards_Asterisk_Asterisk_Colon_0, fileSpec)); + function getWildcardDirectories(includes: string[], path: string, useCaseSensitiveFileNames: boolean) { + // We watch a directory recursively if it contains a wildcard anywhere in a directory segment + // of the pattern: + // + // /a/b/**/d - Watch /a/b recursively to catch changes to any d in any subfolder recursively + // /a/b/*/d - Watch /a/b recursively to catch any d in any immediate subfolder, even if a new subfolder is added + // + // We watch a directory without recursion if it contains a wildcard in the file segment of + // the pattern: + // + // /a/b/* - Watch /a/b directly to catch any new file + // /a/b/a?z - Watch /a/b directly to catch any new file matching a?z + const wildcardDirectories: Map = {}; + if (includes !== undefined) { + const recursiveKeys: string[] = []; + for (const include of includes) { + const name = combinePaths(path, include); + const match = wildcardDirectoryPattern.exec(name); + if (match) { + const key = useCaseSensitiveFileNames ? match[0] : match[0].toLowerCase(); + const flags = watchRecursivePattern.test(name) ? WatchDirectoryFlags.Recursive : WatchDirectoryFlags.None; + const existingFlags = getProperty(wildcardDirectories, key); + if (existingFlags === undefined || existingFlags < flags) { + wildcardDirectories[key] = flags; + if (flags === WatchDirectoryFlags.Recursive) { + recursiveKeys.push(key); + } + } } - - return ExpansionState.Error; } - if (offset >= fileSpec.length) { - // If there is no file specification following the recursive directory pattern - // then we report an error as we cannot match any files. - if (errors) { - errors.push(createCompilerDiagnostic(Diagnostics.File_specification_cannot_end_in_a_recursive_directory_wildcard_Asterisk_Asterisk_Colon_0, fileSpec)); - } - - return ExpansionState.Error; - } - - // Keep track of the recursive wildcard directory - wildcardDirectories.set(basePath, WatchDirectoryFlags.Recursive); - - // Expand the recursive directory pattern. - return expandRecursiveDirectory(fileSpec, basePath, offset + 1, context); - } - - if (!isExpandingRecursiveDirectory) { - wildcardDirectories.set(basePath, WatchDirectoryFlags.None); - } - - // Match the entries in the directory against the wildcard pattern. - const pattern = createRegularExpressionFromWildcard(fileSpec, start, offset, host); - - // If there are no more directory separators (the offset is at the end of the file specification), then - // this must be a file. - if (offset >= fileSpec.length) { - const wildcardHasExtension = fileSegmentHasExtension(fileSpec, start); - const fileNames = readDirectory(basePath, FileSystemEntryKind.File, context); - for (const fileName of fileNames) { - // Skip the file if it doesn't match the pattern. - if (!pattern.test(fileName)) { - continue; - } - - const path = toPath(fileName, basePath, caseSensitiveKeyMapper); - - // If we have excluded this path, we should skip the file. - if (isExcludedPath(path, excludePattern)) { - continue; - } - - // This wildcard has no further directory to process, so include the file. - includeFile(path, context, wildcardHasExtension); - } - } - else { - const directoryNames = readDirectory(basePath, FileSystemEntryKind.Directory, context); - for (const directoryName of directoryNames) { - if (pattern.test(directoryName)) { - const newBasePath = toPath(directoryName, basePath, caseSensitiveKeyMapper); - - // Expand the entries in this directory. - if (expandFileSpec(fileSpec, newBasePath, offset + 1, context, isExpandingRecursiveDirectory) === ExpansionState.Error) { - return ExpansionState.Error; + // Remove any subpaths under an existing recursively watched directory. + for (const key in wildcardDirectories) { + if (hasProperty(wildcardDirectories, key)) { + for (const recursiveKey in recursiveKeys) { + if (containsPath(recursiveKey, key, path, !useCaseSensitiveFileNames)) { + delete wildcardDirectories[key]; + } } } } } - return ExpansionState.Ok; - } - - /** - * Expands a `**` recursive directory wildcard. - * - * @param fileSpec The original file specification. - * @param basePath The directory to recursively expand. - * @param start The starting offset in the file specification. - * @param context The expansion context. - */ - function expandRecursiveDirectory(fileSpec: string, basePath: Path, start: number, context: ExpansionContext): ExpansionState { - // Skip the directory if it is excluded. - if (isExcludedPath(basePath, context.excludePattern)) { - return ExpansionState.Ok; - } - - // Expand the non-recursive part of the file specification against the prefix path. - if (expandFileSpec(fileSpec, basePath, start, context, /*isExpandingRecursiveDirectory*/ true) === ExpansionState.Error) { - return ExpansionState.Error; - } - - // Recursively expand each subdirectory. - const directoryNames = readDirectory(basePath, FileSystemEntryKind.Directory, context); - for (const directoryName of directoryNames) { - const newBasePath = toPath(directoryName, basePath, caseSensitiveKeyMapper); - if (expandRecursiveDirectory(fileSpec, newBasePath, start, context) === ExpansionState.Error) { - return ExpansionState.Error; - } - } - - return ExpansionState.Ok; - } - - /** - * Attempts to include a file in a file set. - * - * @param file The file to include. - * @param context The expansion context. - * @param wildcardHasExtension A value indicating whether the wildcard supplied an explicit extension. - */ - function includeFile(file: Path, context: ExpansionContext, wildcardHasExtension: boolean): void { - const { options, literalFiles, wildcardFiles, excludePattern, supportedExtensions } = context; - - // Ignore the file if it does not have a supported extension. - if ((!wildcardHasExtension || !options.allowNonTsExtensions) && !isSupportedSourceFileName(file, options)) { - return; - } - - // If we have already included a literal or wildcard path with a - // higher priority extension, we should skip this file. - // - // This handles cases where we may encounter both .ts and - // .d.ts (or .js if "allowJs" is enabled) in the same - // directory when they are compilation outputs. - const extensionPriority = getExtensionPriority(file, supportedExtensions); - if (hasFileWithHigherPriorityExtension(file, extensionPriority, context)) { - return; - } - - // We may have included a wildcard path with a lower priority - // extension due to the user-defined order of entries in the - // "include" array. If there is a lower priority extension in the - // same directory, we should remove it. - removeWildcardFilesWithLowerPriorityExtension(file, extensionPriority, context); - - if (!literalFiles.contains(file) && !wildcardFiles.contains(file)) { - wildcardFiles.set(file, file); - } - } - - /** - * Tests whether a path exists and is a specific kind of item. Results are - * cached for performance. - * - * @param path The path to tests. - * @param kind The kind of file system entry to find. - * @param context The expansion context. - */ - function pathExists(path: Path, kind: FileSystemEntryKind, context: ExpansionContext) { - const { cache, host } = context; - const entry = cache.get(path); - if (entry === false) { - // If the entry is strictly `false` then the path doesn`t exist, regardless of its kind. - return false; - } - else if (entry === true) { - // If the entry is strictly `true` then a file exists at this path. - return kind === FileSystemEntryKind.File; - } - else if (typeof entry === "object") { - // If the entry is an object, then a directory exists at this path. - return kind === FileSystemEntryKind.Directory; - } - else { - // The entry does not exist in the cache, so we need to check the host. - if (kind === FileSystemEntryKind.File) { - const result = host.fileExists(path); - cache.set(path, result); - return result; - } - else if (kind === FileSystemEntryKind.Directory) { - const result = host.directoryExists(path); - cache.set(path, result ? {} : false); - return result; - } - } - - return false; - } - - /** - * Reads the contents of a directory for a specific kind of item. Results are - * cached for performance. - * - * @param basePath The path to the directory. The path must already exist. - * @param kind The kind of file system entry to find. - * @param context The expansion context. - */ - function readDirectory(basePath: Path, kind: FileSystemEntryKind, context: ExpansionContext) { - const { cache, host } = context; - - let entry = cache.get(basePath); - if (entry === undefined) { - entry = {}; - cache.set(basePath, entry); - } - - if (typeof entry === "object") { - if (kind === FileSystemEntryKind.File) { - if (entry.files === undefined) { - entry.files = host.readFileNames(basePath); - } - - return entry.files; - } - else if (kind === FileSystemEntryKind.Directory) { - if (entry.directories === undefined) { - entry.directories = host.readDirectoryNames(basePath); - } - - return entry.directories; - } - } - - return []; + return wildcardDirectories; } /** @@ -994,13 +750,13 @@ namespace ts { * @param extensionPriority The priority of the extension. * @param context The expansion context. */ - function hasFileWithHigherPriorityExtension(file: Path, extensionPriority: ExtensionPriority, context: ExpansionContext) { - const { literalFiles, wildcardFiles, supportedExtensions } = context; + function hasFileWithHigherPriorityExtension(file: string, literalFiles: Map, wildcardFiles: Map, extensions: string[], keyMapper: (value: string) => string) { + const extensionPriority = getExtensionPriority(file, extensions); const adjustedExtensionPriority = adjustExtensionPriority(extensionPriority); for (let i = ExtensionPriority.Highest; i < adjustedExtensionPriority; ++i) { - const higherPriorityExtension = supportedExtensions[i]; - const higherPriorityPath = changeExtension(file, higherPriorityExtension); - if (literalFiles.contains(higherPriorityPath) || wildcardFiles.contains(higherPriorityPath)) { + const higherPriorityExtension = extensions[i]; + const higherPriorityPath = keyMapper(changeExtension(file, higherPriorityExtension)); + if (hasProperty(literalFiles, higherPriorityPath) || hasProperty(wildcardFiles, higherPriorityPath)) { return true; } } @@ -1016,13 +772,13 @@ namespace ts { * @param extensionPriority The priority of the extension. * @param context The expansion context. */ - function removeWildcardFilesWithLowerPriorityExtension(file: Path, extensionPriority: ExtensionPriority, context: ExpansionContext) { - const { wildcardFiles, supportedExtensions } = context; + function removeWildcardFilesWithLowerPriorityExtension(file: string, wildcardFiles: Map, extensions: string[], keyMapper: (value: string) => string) { + const extensionPriority = getExtensionPriority(file, extensions); const nextExtensionPriority = getNextLowestExtensionPriority(extensionPriority); - for (let i = nextExtensionPriority; i < supportedExtensions.length; ++i) { - const lowerPriorityExtension = supportedExtensions[i]; - const lowerPriorityPath = changeExtension(file, lowerPriorityExtension); - wildcardFiles.remove(lowerPriorityPath); + for (let i = nextExtensionPriority; i < extensions.length; ++i) { + const lowerPriorityExtension = extensions[i]; + const lowerPriorityPath = keyMapper(changeExtension(file, lowerPriorityExtension)); + delete wildcardFiles[lowerPriorityPath]; } } @@ -1037,297 +793,6 @@ namespace ts { return output; } - /** - * Adds a watched directory to an output map. - * - * @param output The output map. - * @param flags The directory flags. - * @param directory The directory path. - */ - function addDirectoryToOutput(output: Map, flags: WatchDirectoryFlags, directory: string) { - output[directory] = flags; - return output; - } - - /** - * Determines whether a path should be excluded. - * - * @param path The path to test for exclusion. - * @param excludePattern A pattern used to exclude a file specification. - */ - function isExcludedPath(path: string, excludePattern: RegExp) { - return excludePattern ? excludePattern.test(path) : false; - } - - /** - * Determines whether a file segment contains a valid extension. - * - * @param fileSpec The file specification. - * @param segmentStart The offset to the start of the file segment in the specification. - */ - function fileSegmentHasExtension(fileSpec: string, segmentStart: number) { - // if the final path segment does not have a . token, the file does not have an extension. - if (fileSpec.indexOf(".", segmentStart) === -1) { - return false; - } - - // if the extension for the final path segment is (".*"), then the file does not have an extension. - if (fileExtensionIs(fileSpec, ".*")) { - return false; - } - - return true; - } - - /** - * Creates a regular expression from a glob-style wildcard. - * - * @param fileSpec The file specification. - * @param start The starting offset in the file specification. - * @param end The end offset in the file specification. - * @param host The host used to resolve files and directories. - */ - function createRegularExpressionFromWildcard(fileSpec: string, start: number, end: number, host: ParseConfigHost): RegExp { - const pattern = createPatternFromWildcard(fileSpec, start, end); - return new RegExp("^" + pattern + "$", host.useCaseSensitiveFileNames ? "" : "i"); - } - - /** - * Creates a pattern from a wildcard segment. - * - * @param fileSpec The file specification. - * @param start The starting offset in the file specification. - * @param end The end offset in the file specification. - */ - function createPatternFromWildcard(fileSpec: string, start: number, end: number): string { - let pattern = ""; - let offset = indexOfWildcard(fileSpec, start); - while (offset >= 0 && offset < end) { - if (offset > start) { - // Escape and append the non-wildcard portion to the regular expression. - pattern += escapeRegularExpressionText(fileSpec, start, offset); - } - - const charCode = fileSpec.charCodeAt(offset); - if (charCode === CharacterCodes.asterisk) { - // Append a multi-character (zero or more characters) pattern to the regular expression. - pattern += "[^/]*?"; - } - else if (charCode === CharacterCodes.question) { - // Append a single-character pattern to the regular expression. - pattern += "[^/]"; - } - - start = offset + 1; - offset = indexOfWildcard(fileSpec, start); - } - - // Escape and append any remaining non-wildcard portion. - if (start < end) { - pattern += escapeRegularExpressionText(fileSpec, start, end); - } - - return pattern; - } - - /** - * Creates a regular expression from a glob-style wildcard used to exclude a file. - * - * @param excludeSpecs The file specifications to exclude. - * @param basePath The prefix path. - * @param options Compiler options. - * @param host The host used to resolve files and directories. - * @param errors An array for diagnostic reporting. - */ - function createExcludeRegularExpression(excludeSpecs: string[], basePath: string, options: CompilerOptions, host: ParseConfigHost, errors: Diagnostic[]): RegExp { - // Ignore an empty exclusion list - if (!excludeSpecs || excludeSpecs.length === 0) { - return undefined; - } - - basePath = escapeRegularExpressionText(basePath, 0, basePath.length); - - let pattern = ""; - for (const excludeSpec of excludeSpecs) { - const excludePattern = createExcludePattern(excludeSpec, basePath, options, host, errors); - if (excludePattern) { - if (pattern.length > 0) { - pattern += "|"; - } - - pattern += "(" + excludePattern + ")"; - } - } - - if (pattern.length > 0) { - return new RegExp("^(" + pattern + ")($|/)", host.useCaseSensitiveFileNames ? "" : "i"); - } - - return undefined; - } - - /** - * Creates a pattern for used to exclude a file. - * - * @param excludeSpec The file specification to exclude. - * @param basePath The base path for the exclude pattern. - * @param options Compiler options. - * @param host The host used to resolve files and directories. - * @param errors An array for diagnostic reporting. - */ - function createExcludePattern(excludeSpec: string, basePath: string, options: CompilerOptions, host: ParseConfigHost, errors: Diagnostic[]): string { - if (!excludeSpec) { - return undefined; - } - - excludeSpec = normalizePath(excludeSpec); - excludeSpec = removeTrailingDirectorySeparator(excludeSpec); - - let pattern = isRootedDiskPath(excludeSpec) ? "" : basePath; - let hasRecursiveDirectoryWildcard = false; - let segmentStart = 0; - let segmentEnd = getEndOfPathSegment(excludeSpec, segmentStart); - while (segmentStart < segmentEnd) { - if (isRecursiveDirectoryWildcard(excludeSpec, segmentStart, segmentEnd)) { - if (hasRecursiveDirectoryWildcard) { - if (errors) { - errors.push(createCompilerDiagnostic(Diagnostics.File_specification_cannot_contain_multiple_recursive_directory_wildcards_Asterisk_Asterisk_Colon_0, excludeSpec)); - } - - return undefined; - } - - // As an optimization, if the recursive directory is the last - // wildcard, or is followed by only `*` or `*.ts`, don't add the - // remaining pattern and exit the loop. - if (canElideRecursiveDirectorySegment(excludeSpec, segmentEnd, options, host)) { - break; - } - - hasRecursiveDirectoryWildcard = true; - pattern += "(/.+)?"; - } - else { - if (pattern) { - pattern += directorySeparator; - } - - pattern += createPatternFromWildcard(excludeSpec, segmentStart, segmentEnd); - } - - segmentStart = segmentEnd + 1; - segmentEnd = getEndOfPathSegment(excludeSpec, segmentStart); - } - - return pattern; - } - - /** - * Determines whether a recursive directory segment can be elided when - * building a regular expression to exclude a path. - * - * @param excludeSpec The file specification used to exclude a path. - * @param segmentEnd The end position of the recursive directory segment. - * @param options Compiler options. - * @param host The host used to resolve files and directories. - */ - function canElideRecursiveDirectorySegment(excludeSpec: string, segmentEnd: number, options: CompilerOptions, host: ParseConfigHost) { - // If there are no segments after this segment, the pattern for this segment may be elided. - if (segmentEnd + 1 >= excludeSpec.length) { - return true; - } - - // If the following segment is a wildcard that may be elided, the pattern for this segment may be elided. - return canElideWildcardSegment(excludeSpec, segmentEnd + 1, options, host); - } - - /** - * Determines whether a wildcard segment can be elided when building a - * regular expression to exclude a path. - * - * @param excludeSpec The file specification used to exclude a path. - * @param segmentStart The starting position of the segment. - * @param options Compiler options. - * @param host The host used to resolve files and directories. - */ - function canElideWildcardSegment(excludeSpec: string, segmentStart: number, options: CompilerOptions, host: ParseConfigHost) { - const charCode = excludeSpec.charCodeAt(segmentStart); - if (charCode === CharacterCodes.asterisk) { - const end = excludeSpec.length; - - // If the segment consists only of `*`, we may elide this segment. - if (segmentStart + 1 === end) { - return true; - } - - // If the segment consists only of `*.ts`, and we do not allow - // any other extensions for source files, we may elide this segment. - if (!options.allowNonTsExtensions && !options.jsx && !options.allowJs && segmentStart + 4 === end) { - const segment = excludeSpec.substr(segmentStart); - return fileExtensionIs(host.useCaseSensitiveFileNames ? segment : segment.toLowerCase(), ".ts"); - } - } - return false; - } - - /** - * Escape regular expression reserved tokens. - * - * @param text The text to escape. - * @param start The starting offset in the string. - * @param end The ending offset in the string. - */ - function escapeRegularExpressionText(text: string, start: number, end: number) { - return text.substring(start, end).replace(reservedCharacterPattern, "\\$&"); - } - - /** - * Determines whether the wildcard at the current offset is a recursive directory wildcard. - * - * @param fileSpec The file specification. - * @param segmentStart The starting offset of a segment in the file specification. - * @param segmentEnd The ending offset of a segment in the file specification. - */ - function isRecursiveDirectoryWildcard(fileSpec: string, segmentStart: number, segmentEnd: number) { - return segmentEnd - segmentStart === 2 && - fileSpec.charCodeAt(segmentStart) === CharacterCodes.asterisk && - fileSpec.charCodeAt(segmentStart + 1) === CharacterCodes.asterisk; - } - - /** - * Gets the index of the next wildcard character in a file specification. - * - * @param fileSpec The file specification. - * @param start The starting offset in the file specification. - */ - function indexOfWildcard(fileSpec: string, start: number, end: number = fileSpec.length): number { - for (let i = start; i < end; ++i) { - const ch = fileSpec.charCodeAt(i); - if (ch === CharacterCodes.asterisk || ch === CharacterCodes.question) { - return i; - } - } - - return -1; - } - - /** - * Get the end position of a path segment, either the index of the next directory separator or - * the provided end position. - * - * @param fileSpec The file specification. - * @param segmentStart The start offset in the file specification. - */ - function getEndOfPathSegment(fileSpec: string, segmentStart: number): number { - const end = fileSpec.length; - if (segmentStart >= end) { - return end; - } - - const offset = fileSpec.indexOf(directorySeparator, segmentStart); - return offset < 0 ? end : offset; - } - /** * Gets a case sensitive key. * diff --git a/src/compiler/core.ts b/src/compiler/core.ts index 3e6981996e1..465d4d229d8 100644 --- a/src/compiler/core.ts +++ b/src/compiler/core.ts @@ -25,9 +25,7 @@ namespace ts { contains, remove, forEachValue: forEachValueInMap, - reduceProperties: reducePropertiesInMap, clear, - mergeFrom }; function forEachValueInMap(f: (key: Path, value: T) => void) { @@ -36,10 +34,6 @@ namespace ts { } } - function reducePropertiesInMap(callback: (memo: U, value: T, key: Path) => U, initial: U) { - return reduceProperties(files, callback, initial); - } - // path should already be well-formed so it does not need to be normalized function get(path: Path): T { return files[toKey(path)]; @@ -62,16 +56,6 @@ namespace ts { files = {}; } - function mergeFrom(other: FileMap) { - other.forEachValue(mergeFromOther); - } - - function mergeFromOther(key: Path, value: T) { - if (!contains(key)) { - set(key, value); - } - } - function toKey(path: Path): string { return keyMapper ? keyMapper(path) : path; } @@ -131,6 +115,15 @@ namespace ts { return -1; } + export function indexOfAnyCharCode(text: string, charCodes: number[], start?: number): number { + for (let i = start || 0, len = text.length; i < len; ++i) { + if (contains(charCodes, text.charCodeAt(i))) { + return i; + } + } + return -1; + } + export function countWhere(array: T[], predicate: (x: T) => boolean): number { let count = 0; if (array) { @@ -524,6 +517,10 @@ namespace ts { return a < b ? Comparison.LessThan : Comparison.GreaterThan; } + export function compareStringsCaseInsensitive(a: string, b: string) { + return compareStrings(a, b, /*ignoreCase*/ true); + } + function getDiagnosticFileName(diagnostic: Diagnostic): string { return diagnostic.file ? diagnostic.file.fileName : undefined; } @@ -861,6 +858,180 @@ namespace ts { return pathLen > extLen && path.substr(pathLen - extLen, extLen) === extension; } + export function fileExtensionIsAny(path: string, extensions: string[]): boolean { + for (const extension of extensions) { + if (fileExtensionIs(path, extension)) { + return true; + } + } + + return false; + } + + + // Reserved characters, forces escaping of any non-word (or digit), non-whitespace character. + // It may be inefficient (we could just match (/[-[\]{}()*+?.,\\^$|#\s]/g), but this is future + // proof. + const reservedCharacterPattern = /[^\w\s\/]/g; + const wildcardCharCodes = [CharacterCodes.asterisk, CharacterCodes.question]; + + export function getRegularExpressionForWildcard(specs: string[], basePath: string, usage: "files" | "directories" | "exclude", useCaseSensitiveFileNames: boolean) { + if (specs === undefined || specs.length === 0) { + return undefined; + } + + let pattern = ""; + let hasWrittenSubpattern = false; + spec: for (const spec of specs) { + if (!spec) { + continue; + } + + let subpattern = ""; + let hasRecursiveDirectoryWildcard = false; + let hasWrittenComponent = false; + const components = getNormalizedPathComponents(spec, basePath); + if (usage !== "exclude" && components[components.length - 1] === "**") { + continue spec; + } + + // getNormalizedPathComponents includes the separator for the root component. + // We need to remove to create our regex correctly. + components[0] = removeTrailingDirectorySeparator(components[0]); + + let optionalCount = 0; + for (const component of components) { + if (component === "**") { + if (hasRecursiveDirectoryWildcard) { + continue spec; + } + + subpattern += "(/.+?)?"; + hasRecursiveDirectoryWildcard = true; + hasWrittenComponent = true; + } + else { + if (usage === "directories") { + subpattern += "("; + optionalCount++; + } + + if (hasWrittenComponent) { + subpattern += directorySeparator; + } + + subpattern += component.replace(reservedCharacterPattern, replaceWildcardCharacter); + hasWrittenComponent = true; + } + } + + while (optionalCount > 0) { + subpattern += ")?"; + optionalCount--; + } + + if (hasWrittenSubpattern) { + pattern += "|"; + } + + pattern += "(" + subpattern + ")"; + hasWrittenSubpattern = true; + } + + if (!pattern) { + return undefined; + } + + return new RegExp("^(" + pattern + (usage === "exclude" ? ")($|/)" : ")$"), useCaseSensitiveFileNames ? "" : "i"); + } + + function replaceWildcardCharacter(match: string) { + return match === "*" ? "[^/]*" : match === "?" ? "[^/]" : "\\" + match; + } + + export interface FileSystemEntries { + files: string[]; + directories: string[]; + } + + export function matchFiles(path: string, extensions: string[], excludes: string[], includes: string[], useCaseSensitiveFileNames: boolean, currentDirectory: string, getFileSystemEntries: (path: string) => FileSystemEntries): string[] { + path = normalizePath(path); + currentDirectory = normalizePath(currentDirectory); + const absolutePath = combinePaths(currentDirectory, path); + const includeFileRegex = getRegularExpressionForWildcard(includes, absolutePath, "files", useCaseSensitiveFileNames); + const includeDirectoryRegex = getRegularExpressionForWildcard(includes, absolutePath, "directories", useCaseSensitiveFileNames); + const excludeRegex = getRegularExpressionForWildcard(excludes, absolutePath, "exclude", useCaseSensitiveFileNames); + const result: string[] = []; + for (const basePath of getBasePaths(path, includes, useCaseSensitiveFileNames)) { + visitDirectory(basePath, combinePaths(currentDirectory, basePath)); + } + return result; + + function visitDirectory(path: string, absolutePath: string) { + const { files, directories } = getFileSystemEntries(path); + + for (const current of files) { + const name = combinePaths(path, current); + const absoluteName = combinePaths(absolutePath, current); + if ((!extensions || fileExtensionIsAny(name, extensions)) && + (!includeFileRegex || includeFileRegex.test(absoluteName)) && + (!excludeRegex || !excludeRegex.test(absoluteName))) { + result.push(name); + } + } + + for (const current of directories) { + const name = combinePaths(path, current); + const absoluteName = combinePaths(absolutePath, current); + if ((!includeDirectoryRegex || includeDirectoryRegex.test(absoluteName)) && + (!excludeRegex || !excludeRegex.test(absoluteName))) { + visitDirectory(name, absoluteName); + } + } + } + } + + /** + * Computes the unique non-wildcard base paths amongst the provided include patterns. + */ + function getBasePaths(path: string, includes: string[], useCaseSensitiveFileNames: boolean) { + // Storage for our results in the form of literal paths (e.g. the paths as written by the user). + const basePaths: string[] = [path]; + if (includes) { + // Storage for literal base paths amongst the include patterns. + const includeBasePaths: string[] = []; + for (const include of includes) { + if (isRootedDiskPath(include)) { + const wildcardOffset = indexOfAnyCharCode(include, wildcardCharCodes); + const includeBasePath = wildcardOffset < 0 + ? removeTrailingDirectorySeparator(getDirectoryPath(include)) + : include.substring(0, include.lastIndexOf(directorySeparator, wildcardOffset)); + + // Append the literal and canonical candidate base paths. + includeBasePaths.push(includeBasePath); + } + } + + // Sort the offsets array using either the literal or canonical path representations. + includeBasePaths.sort(useCaseSensitiveFileNames ? compareStrings : compareStringsCaseInsensitive); + + // Iterate over each include base path and include unique base paths that are not a + // subpath of an existing base path + include: for (let i = 0; i < includeBasePaths.length; ++i) { + const includeBasePath = includeBasePaths[i]; + for (let j = 0; j < basePaths.length; ++j) { + if (containsPath(basePaths[j], includeBasePath, path, !useCaseSensitiveFileNames)) { + continue include; + } + } + + basePaths.push(includeBasePath); + } + } + + return basePaths; + } + /** * List of supported extensions in order of file resolution precedence. */ diff --git a/src/compiler/sys.ts b/src/compiler/sys.ts index 5ff8dbf6796..cad64adb4cf 100644 --- a/src/compiler/sys.ts +++ b/src/compiler/sys.ts @@ -16,9 +16,7 @@ namespace ts { createDirectory(path: string): void; getExecutingFilePath(): string; getCurrentDirectory(): string; - readDirectory(path: string, extension?: string, exclude?: string[]): string[]; - readFileNames(path: string): string[]; - readDirectoryNames(path: string): string[]; + readDirectory(path: string, extensions?: string[], exclude?: string[], include?: string[]): string[]; getMemoryUsage?(): number; exit(exitCode?: number): void; } @@ -61,9 +59,7 @@ namespace ts { resolvePath(path: string): string; readFile(path: string): string; writeFile(path: string, contents: string): void; - readDirectory(path: string, extension?: string, exclude?: string[]): string[]; - readDirectoryNames(path: string): string[]; - readFileNames(path: string): string[]; + readDirectory(path: string, extensions?: string[], exclude?: string[], include?: string[]): string[]; }; export var sys: System = (function () { @@ -71,6 +67,7 @@ namespace ts { function getWScriptSystem(): System { const fso = new ActiveXObject("Scripting.FileSystemObject"); + const shell = new ActiveXObject("WScript.Shell"); const fileStream = new ActiveXObject("ADODB.Stream"); fileStream.Type = 2 /*text*/; @@ -150,38 +147,20 @@ namespace ts { return result.sort(); } - function readDirectory(path: string, extension?: string, exclude?: string[]): string[] { - const result: string[] = []; - exclude = map(exclude, s => getCanonicalPath(combinePaths(path, s))); - visitDirectory(path); - return result; - function visitDirectory(path: string) { + function getAccessibleFileSystemEntries(path: string): FileSystemEntries { + try { const folder = fso.GetFolder(path || "."); const files = getNames(folder.files); - for (const current of files) { - const name = combinePaths(path, current); - if ((!extension || fileExtensionIs(name, extension)) && !contains(exclude, getCanonicalPath(name))) { - result.push(name); - } - } - const subfolders = getNames(folder.subfolders); - for (const current of subfolders) { - const name = combinePaths(path, current); - if (!contains(exclude, getCanonicalPath(name))) { - visitDirectory(name); - } - } + const directories = getNames(folder.subfolders); + return { files, directories }; + } + catch (e) { + return { files: [], directories: [] }; } } - function readFileNames(path: string): string[] { - const folder = fso.GetFolder(path || "."); - return getNames(folder.files); - } - - function readDirectoryNames(path: string): string[] { - const folder = fso.GetFolder(path || "."); - return getNames(folder.directories); + function readDirectory(path: string, extensions?: string[], excludes?: string[], includes?: string[]): string[] { + return matchFiles(path, extensions, excludes, includes, /*useCaseSensitiveFileNames*/ false, shell.CurrentDirectory, getAccessibleFileSystemEntries); } return { @@ -211,11 +190,9 @@ namespace ts { return WScript.ScriptFullName; }, getCurrentDirectory() { - return new ActiveXObject("WScript.Shell").CurrentDirectory; + return shell.CurrentDirectory; }, readDirectory, - readFileNames, - readDirectoryNames, exit(exitCode?: number): void { try { WScript.Quit(exitCode); @@ -385,56 +362,30 @@ namespace ts { return useCaseSensitiveFileNames ? path.toLowerCase() : path; } - function readDirectory(path: string, extension?: string, exclude?: string[]): string[] { - const result: string[] = []; - exclude = map(exclude, s => getCanonicalPath(combinePaths(path, s))); - visitDirectory(path); - return result; - function visitDirectory(path: string) { - const files = _fs.readdirSync(path || ".").sort(); + function getAccessibleFileSystemEntries(path: string): FileSystemEntries { + try { + const entries = _fs.readdirSync(path || ".").sort(); + const files: string[] = []; const directories: string[] = []; - for (const current of files) { - const name = combinePaths(path, current); - if (!contains(exclude, getCanonicalPath(name))) { - const stat = _fs.statSync(name); - if (stat.isFile()) { - if (!extension || fileExtensionIs(name, extension)) { - result.push(name); - } - } - else if (stat.isDirectory()) { - directories.push(name); - } + for (const entry of entries) { + const name = combinePaths(path, entry); + const stat = _fs.statSync(name); + if (stat.isFile()) { + files.push(entry); + } + else if (stat.isDirectory()) { + directories.push(entry); } } - for (const current of directories) { - visitDirectory(current); - } + return { files, directories }; + } + catch (e) { + return { files: [], directories: [] }; } } - function readFileNames(path: string): string[] { - const entries = _fs.readdirSync(path || "."); - const files: string[] = []; - for (const entry of entries) { - const stat = _fs.statSync(combinePaths(path, entry)); - if (stat.isFile()) { - files.push(entry); - } - } - return files.sort(); - } - - function readDirectoryNames(path: string): string[] { - const entries = _fs.readdirSync(path || "."); - const directories: string[] = []; - for (const entry of entries) { - const stat = _fs.statSync(combinePaths(path, entry)); - if (stat.isDirectory()) { - directories.push(entry); - } - } - return directories.sort(); + function readDirectory(path: string, extensions?: string[], excludes?: string[], includes?: string[]): string[] { + return matchFiles(path, extensions, excludes, includes, useCaseSensitiveFileNames, process.cwd(), getAccessibleFileSystemEntries); } return { @@ -482,7 +433,7 @@ namespace ts { return _path.resolve(path); }, fileExists(path: string): boolean { - return _fs.existsSync(path) && _fs.statSync(path).isFile(); + return _fs.existsSync(path); }, directoryExists(path: string) { return _fs.existsSync(path) && _fs.statSync(path).isDirectory(); @@ -499,8 +450,6 @@ namespace ts { return process.cwd(); }, readDirectory, - readFileNames, - readDirectoryNames, getMemoryUsage() { if (global.gc) { global.gc(); @@ -539,8 +488,6 @@ namespace ts { getExecutingFilePath: () => ChakraHost.executingFile, getCurrentDirectory: () => ChakraHost.currentDirectory, readDirectory: ChakraHost.readDirectory, - readFileNames: ChakraHost.readFileNames, - readDirectoryNames: ChakraHost.readDirectoryNames, exit: ChakraHost.quit, }; } @@ -560,6 +507,4 @@ namespace ts { return undefined; // Unsupported host } })(); -} - - +} \ No newline at end of file diff --git a/src/compiler/types.ts b/src/compiler/types.ts index 4c62904f4e8..3ddc0b447a9 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -14,8 +14,6 @@ namespace ts { remove(fileName: Path): void; forEachValue(f: (key: Path, v: T) => void): void; - reduceProperties(f: (memo: U, value: T, key: Path) => U, initial: U): U; - mergeFrom(other: FileMap): void; clear(): void; } @@ -1586,31 +1584,13 @@ namespace ts { export interface ParseConfigHost { useCaseSensitiveFileNames: boolean; - readDirectory(rootDir: string, extension: string, exclude: string[]): string[]; + readDirectory(rootDir: string, extensions: string[], excludes: string[], includes: string[]): string[]; /** * Gets a value indicating whether the specified path exists and is a file. * @param path The path to test. */ fileExists(path: string): boolean; - - /** - * Gets a value indicating whether the specified path exists and is a directory. - * @param path The path to test. - */ - directoryExists(path: string): boolean; - - /** - * Reads the files names in the directory. - * @param rootDir The directory path. - */ - readFileNames(rootDir: string): string[]; - - /** - * Reads the directory names in the directory. - * @param rootDir The directory path. - */ - readDirectoryNames(rootDir: string): string[]; } export interface WriteFileCallback { diff --git a/src/harness/harness.ts b/src/harness/harness.ts index d8ccc512343..509c193666f 100644 --- a/src/harness/harness.ts +++ b/src/harness/harness.ts @@ -23,6 +23,7 @@ /// /// /// +/// /* tslint:disable:no-null */ // Block scoped definitions work poorly for global variables, temporarily enable var @@ -435,9 +436,7 @@ namespace Harness { args(): string[]; getExecutingFilePath(): string; exit(exitCode?: number): void; - readDirectory(path: string, extension?: string, exclude?: string[]): string[]; - readDirectoryNames(path: string): string[]; - readFileNames(path: string): string[]; + readDirectory(path: string, extension?: string[], exclude?: string[], include?: string[]): string[]; } export var IO: IO; @@ -475,9 +474,7 @@ namespace Harness { export const directoryExists: typeof IO.directoryExists = fso.FolderExists; export const fileExists: typeof IO.fileExists = fso.FileExists; export const log: typeof IO.log = global.WScript && global.WScript.StdOut.WriteLine; - export const readDirectory: typeof IO.readDirectory = (path, extension, exclude) => ts.sys.readDirectory(path, extension, exclude); - export const readDirectoryNames: typeof IO.readDirectoryNames = path => ts.sys.readDirectoryNames(path); - export const readFileNames: typeof IO.readFileNames = path => ts.sys.readFileNames(path); + export const readDirectory: typeof IO.readDirectory = (path, extension, exclude, include) => ts.sys.readDirectory(path, extension, exclude, include); export function createDirectory(path: string) { if (directoryExists(path)) { @@ -547,9 +544,7 @@ namespace Harness { export const fileExists: typeof IO.fileExists = fs.existsSync; export const log: typeof IO.log = s => console.log(s); - export const readDirectory: typeof IO.readDirectory = (path, extension, exclude) => ts.sys.readDirectory(path, extension, exclude); - export const readDirectoryNames: typeof IO.readDirectoryNames = path => ts.sys.readDirectoryNames(path); - export const readFileNames: typeof IO.readFileNames = path => ts.sys.readFileNames(path); + export const readDirectory: typeof IO.readDirectory = (path, extension, exclude, include) => ts.sys.readDirectory(path, extension, exclude, include); export function createDirectory(path: string) { if (!directoryExists(path)) { @@ -755,16 +750,22 @@ namespace Harness { Http.writeToServerSync(serverRoot + path, "WRITE", contents); } - export function readDirectory(path: string, extension?: string, exclude?: string[]) { - return listFiles(path).filter(f => !extension || ts.fileExtensionIs(f, extension)); - } - - export function readDirectoryNames(path: string): string[] { - return []; - } - - export function readFileNames(path: string) { - return readDirectory(path); + export function readDirectory(path: string, extension?: string[], exclude?: string[], include?: string[]) { + const fs = new Utils.VirtualFileSystem(path, useCaseSensitiveFileNames()); + for (const file in listFiles(path)) { + fs.addFile(file); + } + return ts.matchFiles(path, extension, exclude, include, useCaseSensitiveFileNames(), getCurrentDirectory(), path => { + const entry = fs.traversePath(path); + if (entry && entry.isDirectory()) { + const directory = entry; + return { + files: ts.map(directory.getFiles(), f => f.name), + directories: ts.map(directory.getDirectories(), d => d.name) + }; + } + return { files: [], directories: [] }; + }); } } } diff --git a/src/harness/harnessLanguageService.ts b/src/harness/harnessLanguageService.ts index 521d17cfc8f..4db96f344df 100644 --- a/src/harness/harnessLanguageService.ts +++ b/src/harness/harnessLanguageService.ts @@ -575,18 +575,10 @@ namespace Harness.LanguageService { return this.host.getCurrentDirectory(); } - readDirectory(path: string, extension?: string): string[] { + readDirectory(path: string, extension?: string[], exclude?: string[], include?: string[]): string[] { throw new Error("Not implemented Yet."); } - readDirectoryNames(path: string): string[] { - throw new Error("Not implemented."); - } - - readFileNames(path: string): string[] { - throw new Error("Not implemented."); - } - watchFile(fileName: string, callback: (fileName: string) => void): ts.FileWatcher { return { close() { } }; } diff --git a/src/harness/loggedIO.ts b/src/harness/loggedIO.ts index 0bae91f7976..a6df6382bd0 100644 --- a/src/harness/loggedIO.ts +++ b/src/harness/loggedIO.ts @@ -62,8 +62,9 @@ interface IOLog { }[]; directoriesRead: { path: string, - extension: string, + extension: string[], exclude: string[], + include: string[], result: string[] }[]; } @@ -217,9 +218,9 @@ namespace Playback { memoize(path => findResultByPath(wrapper, replayLog.filesRead, path).contents)); wrapper.readDirectory = recordReplay(wrapper.readDirectory, underlying)( - (path, extension, exclude) => { - const result = (underlying).readDirectory(path, extension, exclude); - const logEntry = { path, extension, exclude, result }; + (path, extension, exclude, include) => { + const result = (underlying).readDirectory(path, extension, exclude, include); + const logEntry = { path, extension, exclude, include, result }; recordLog.directoriesRead.push(logEntry); return result; }, diff --git a/src/harness/projectsRunner.ts b/src/harness/projectsRunner.ts index 76c7952428a..bff391cf9af 100644 --- a/src/harness/projectsRunner.ts +++ b/src/harness/projectsRunner.ts @@ -213,10 +213,7 @@ class ProjectRunner extends RunnerBase { const configParseHost: ts.ParseConfigHost = { useCaseSensitiveFileNames: Harness.IO.useCaseSensitiveFileNames(), fileExists, - directoryExists, readDirectory, - readDirectoryNames, - readFileNames }; const configParseResult = ts.parseJsonConfigFileContent(configObject, configParseHost, ts.getDirectoryPath(configFileName), compilerOptions); if (configParseResult.errors.length > 0) { @@ -276,8 +273,8 @@ class ProjectRunner extends RunnerBase { : ts.normalizeSlashes(testCase.projectRoot) + "/" + ts.normalizeSlashes(fileName); } - function readDirectory(rootDir: string, extension: string, exclude: string[]): string[] { - const harnessReadDirectoryResult = Harness.IO.readDirectory(getFileNameInTheProjectTest(rootDir), extension, exclude); + function readDirectory(rootDir: string, extension: string[], exclude: string[], include: string[]): string[] { + const harnessReadDirectoryResult = Harness.IO.readDirectory(getFileNameInTheProjectTest(rootDir), extension, exclude, include); const result: string[] = []; for (let i = 0; i < harnessReadDirectoryResult.length; i++) { result[i] = ts.getRelativePathToDirectoryOrUrl(testCase.projectRoot, harnessReadDirectoryResult[i], @@ -286,14 +283,6 @@ class ProjectRunner extends RunnerBase { return result; } - function readDirectoryNames(path: string) { - return Harness.IO.readDirectoryNames(getFileNameInTheProjectTest(path)); - } - - function readFileNames(path: string) { - return Harness.IO.readFileNames(getFileNameInTheProjectTest(path)); - } - function fileExists(fileName: string): boolean { return Harness.IO.fileExists(getFileNameInTheProjectTest(fileName)); } diff --git a/src/harness/rwcRunner.ts b/src/harness/rwcRunner.ts index 826f6b67260..791c684c79c 100644 --- a/src/harness/rwcRunner.ts +++ b/src/harness/rwcRunner.ts @@ -79,10 +79,7 @@ namespace RWC { const configParseHost: ts.ParseConfigHost = { useCaseSensitiveFileNames: Harness.IO.useCaseSensitiveFileNames(), fileExists: Harness.IO.fileExists, - directoryExists: Harness.IO.directoryExists, readDirectory: Harness.IO.readDirectory, - readDirectoryNames: Harness.IO.readDirectoryNames, - readFileNames: Harness.IO.readFileNames, }; const configParseResult = ts.parseJsonConfigFileContent(parsedTsconfigFileContents.config, configParseHost, ts.getDirectoryPath(tsconfigFile.path)); fileNames = configParseResult.fileNames; diff --git a/src/harness/vfs.ts b/src/harness/vfs.ts new file mode 100644 index 00000000000..66bbd715fbc --- /dev/null +++ b/src/harness/vfs.ts @@ -0,0 +1,160 @@ +/// +namespace Utils { + export class VirtualFileSystemEntry { + fileSystem: VirtualFileSystem; + name: string; + + constructor(fileSystem: VirtualFileSystem, name: string) { + this.fileSystem = fileSystem; + this.name = name; + } + + isDirectory() { return false; } + isFile() { return false; } + isFileSystem() { return false; } + } + + export class VirtualFile extends VirtualFileSystemEntry { + content: string; + isFile() { return true; } + } + + export abstract class VirtualFileSystemContainer extends VirtualFileSystemEntry { + abstract getFileSystemEntries(): VirtualFileSystemEntry[]; + + getFileSystemEntry(name: string): VirtualFileSystemEntry { + for (const entry of this.getFileSystemEntries()) { + if (this.fileSystem.sameName(entry.name, name)) { + return entry; + } + } + return undefined; + } + + getDirectories(): VirtualDirectory[] { + return ts.filter(this.getFileSystemEntries(), entry => entry.isDirectory()); + } + + getFiles(): VirtualFile[] { + return ts.filter(this.getFileSystemEntries(), entry => entry.isFile()); + } + + getDirectory(name: string): VirtualDirectory { + const entry = this.getFileSystemEntry(name); + return entry.isDirectory() ? entry : undefined; + } + + getFile(name: string): VirtualFile { + const entry = this.getFileSystemEntry(name); + return entry.isFile() ? entry : undefined; + } + } + + export class VirtualDirectory extends VirtualFileSystemContainer { + private entries: VirtualFileSystemEntry[] = []; + + isDirectory() { return true; } + + getFileSystemEntries() { return this.entries.slice(); } + + addDirectory(name: string): VirtualDirectory { + const entry = this.getFileSystemEntry(name); + if (entry === undefined) { + const directory = new VirtualDirectory(this.fileSystem, name); + this.entries.push(directory); + return directory; + } + else if (entry.isDirectory()) { + return entry; + } + else { + return undefined; + } + } + + addFile(name: string, content?: string): VirtualFile { + const entry = this.getFileSystemEntry(name); + if (entry === undefined) { + const file = new VirtualFile(this.fileSystem, name); + file.content = content; + this.entries.push(file); + return file; + } + else if (entry.isFile()) { + const file = entry; + file.content = content; + return file; + } + else { + return undefined; + } + } + } + + export class VirtualFileSystem extends VirtualFileSystemContainer { + private root: VirtualDirectory; + + currentDirectory: string; + useCaseSensitiveFileNames: boolean; + + constructor(currentDirectory: string, useCaseSensitiveFileNames: boolean) { + super(undefined, ""); + this.fileSystem = this; + this.root = new VirtualDirectory(this, ""); + this.currentDirectory = currentDirectory; + this.useCaseSensitiveFileNames = useCaseSensitiveFileNames; + } + + isFileSystem() { return true; } + + getFileSystemEntries() { return this.root.getFileSystemEntries(); } + + addDirectory(path: string) { + const components = ts.getNormalizedPathComponents(path, this.currentDirectory); + let directory: VirtualDirectory = this.root; + for (const component of components) { + directory = directory.addDirectory(component); + if (directory === undefined) { + break; + } + } + + return directory; + } + + addFile(path: string, content?: string) { + const absolutePath = ts.getNormalizedAbsolutePath(path, this.currentDirectory); + const fileName = ts.getBaseFileName(path); + const directoryPath = ts.getDirectoryPath(absolutePath); + const directory = this.addDirectory(directoryPath); + return directory ? directory.addFile(fileName, content) : undefined; + } + + fileExists(path: string) { + const entry = this.traversePath(path); + return entry !== undefined && entry.isFile(); + } + + sameName(a: string, b: string) { + return this.useCaseSensitiveFileNames ? a === b : a.toLowerCase() === b.toLowerCase(); + } + + traversePath(path: string) { + let directory: VirtualDirectory = this.root; + for (const component of ts.getNormalizedPathComponents(path, this.currentDirectory)) { + const entry = directory.getFileSystemEntry(component); + if (entry === undefined) { + return undefined; + } + else if (entry.isDirectory()) { + directory = entry; + } + else { + return entry; + } + } + + return directory; + } + } +} \ No newline at end of file diff --git a/src/services/shims.ts b/src/services/shims.ts index a7f85d24452..cf6f3885a2c 100644 --- a/src/services/shims.ts +++ b/src/services/shims.ts @@ -72,10 +72,7 @@ namespace ts { * @param exclude A JSON encoded string[] containing the paths to exclude * when enumerating the directory. */ - readDirectory(rootDir: string, extension: string, exclude?: string): string; - readDirectoryNames?(rootDir: string): string; - readFileNames?(rootDir: string): string; - directoryExists?(path: string): boolean; + readDirectory(rootDir: string, extension: string, exclude?: string, include?: string): string; useCaseSensitiveFileNames?: boolean; } @@ -425,70 +422,43 @@ namespace ts { } } - public readDirectory(rootDir: string, extension: string, exclude: string[]): string[] { - // Wrap the API changes for 1.5 release. This try/catch - // should be removed once TypeScript 1.5 has shipped. + public readDirectory(rootDir: string, extensions: string[], exclude: string[], include: string[]): string[] { + // Wrap the API changes for 1.8 release. This try/catch + // should be removed once TypeScript 1.8 has shipped. // Also consider removing the optional designation for // the exclude param at this time. - var encoded: string; try { - encoded = this.shimHost.readDirectory(rootDir, extension, JSON.stringify(exclude)); + return JSON.parse(this.shimHost.readDirectory( + rootDir, + JSON.stringify(extensions), + JSON.stringify(exclude), + JSON.stringify(include))); } catch (e) { - encoded = this.shimHost.readDirectory(rootDir, extension); + let results: string[] = []; + for (const extension of extensions) { + for (const file of this.readDirectoryFallback(rootDir, extension, exclude)) + { + if (!contains(results, file)) { + results.push(file); + } + } + } + return results; } - return JSON.parse(encoded); - } - - public readDirectoryNames(path: string): string[] { - if (this.shimHost.readDirectory) { - const encoded = this.shimHost.readDirectoryNames(path); - return JSON.parse(encoded); - } - - if (sys) { - path = normalizePath(path); - path = ensureTrailingDirectorySeparator(path); - return sys.readDirectoryNames(path); - } - - return []; - } - - public readFileNames(path: string): string[] { - if (this.shimHost.readFileNames) { - const encoded = this.shimHost.readFileNames(path); - return JSON.parse(encoded); - } - - if (sys) { - path = normalizePath(path); - path = ensureTrailingDirectorySeparator(path); - return sys.readFileNames(path); - } - - return []; } public fileExists(fileName: string): boolean { return this.shimHost.fileExists(fileName); } - public directoryExists(directoryName: string): boolean { - if (this.shimHost.directoryExists) { - return this.shimHost.directoryExists(directoryName); - } - - if (sys) { - return sys.directoryExists(directoryName); - } - - return false; - } - public readFile(fileName: string): string { return this.shimHost.readFile(fileName); } + + private readDirectoryFallback(rootDir: string, extension: string, exclude: string[]) { + return JSON.parse(this.shimHost.readDirectory(rootDir, extension, JSON.stringify(exclude))); + } } function simpleForwardCall(logger: Logger, actionDescription: string, action: () => any, logPerformance: boolean): any { diff --git a/tests/cases/unittests/cachingInServerLSHost.ts b/tests/cases/unittests/cachingInServerLSHost.ts index f0a138c27a1..0d2a9ebf54d 100644 --- a/tests/cases/unittests/cachingInServerLSHost.ts +++ b/tests/cases/unittests/cachingInServerLSHost.ts @@ -36,11 +36,9 @@ module ts { getCurrentDirectory: (): string => { return ""; }, - readDirectory: (path: string, extension?: string, exclude?: string[]): string[] => { + readDirectory: (path: string, extension?: string[], exclude?: string[], include?: string[]): string[] => { throw new Error("NYI"); }, - readDirectoryNames: (path: string): string[] => { throw new Error("NYI"); }, - readFileNames: (path: string): string[] => { throw new Error("NYI"); }, exit: (exitCode?: number) => { }, watchFile: (path, callback) => { diff --git a/tests/cases/unittests/expandFiles.ts b/tests/cases/unittests/expandFiles.ts index 3aac938f178..02065407e27 100644 --- a/tests/cases/unittests/expandFiles.ts +++ b/tests/cases/unittests/expandFiles.ts @@ -2,65 +2,91 @@ /// namespace ts { + class MockParseConfigHost extends Utils.VirtualFileSystem implements ParseConfigHost { + constructor(currentDirectory: string, ignoreCase: boolean, files: string[]) { + super(currentDirectory, ignoreCase); + for (const file of files) { + this.addFile(file); + } + } + + readDirectory(path: string, extensions: string[], excludes: string[], includes: string[]) { + return matchFiles(path, extensions, excludes, includes, this.useCaseSensitiveFileNames, this.currentDirectory, (path: string) => this.getAccessibleFileSystemEntries(path)); + } + + getAccessibleFileSystemEntries(path: string) { + const entry = this.traversePath(path); + if (entry && entry.isDirectory()) { + const directory = entry; + return { + files: map(directory.getFiles(), f => f.name), + directories: map(directory.getDirectories(), d => d.name) + }; + } + return { files: [], directories: [] }; + } + } + const caseInsensitiveBasePath = "c:/dev/"; - const caseInsensitiveHost = createMockParseConfigHost(/*ignoreCase*/ true, caseInsensitiveBasePath, [ - "a.ts", - "a.d.ts", - "a.js", - "b.ts", - "b.js", - "c.d.ts", - "z/a.ts", - "z/abz.ts", - "z/aba.ts", - "z/b.ts", - "z/bbz.ts", - "z/bba.ts", - "x/a.ts", - "x/aa.ts", - "x/b.ts", - "x/y/a.ts", - "x/y/b.ts", - "js/a.js", - "js/b.js", + const caseInsensitiveHost = new MockParseConfigHost(caseInsensitiveBasePath, /*useCaseSensitiveFileNames*/ false, [ + "c:/dev/a.ts", + "c:/dev/a.d.ts", + "c:/dev/a.js", + "c:/dev/b.ts", + "c:/dev/b.js", + "c:/dev/c.d.ts", + "c:/dev/z/a.ts", + "c:/dev/z/abz.ts", + "c:/dev/z/aba.ts", + "c:/dev/z/b.ts", + "c:/dev/z/bbz.ts", + "c:/dev/z/bba.ts", + "c:/dev/x/a.ts", + "c:/dev/x/aa.ts", + "c:/dev/x/b.ts", + "c:/dev/x/y/a.ts", + "c:/dev/x/y/b.ts", + "c:/dev/js/a.js", + "c:/dev/js/b.js", + "c:/ext/ext.ts" ]); const caseSensitiveBasePath = "/dev/"; - const caseSensitiveHost = createMockParseConfigHost(/*ignoreCase*/ false, caseSensitiveBasePath, [ - "a.ts", - "a.d.ts", - "a.js", - "b.ts", - "b.js", - "A.ts", - "B.ts", - "c.d.ts", - "z/a.ts", - "z/abz.ts", - "z/aba.ts", - "z/b.ts", - "z/bbz.ts", - "z/bba.ts", - "x/a.ts", - "x/b.ts", - "x/y/a.ts", - "x/y/b.ts", - "js/a.js", - "js/b.js", + const caseSensitiveHost = new MockParseConfigHost(caseSensitiveBasePath, /*useCaseSensitiveFileNames*/ true, [ + "/dev/a.ts", + "/dev/a.d.ts", + "/dev/a.js", + "/dev/b.ts", + "/dev/b.js", + "/dev/A.ts", + "/dev/B.ts", + "/dev/c.d.ts", + "/dev/z/a.ts", + "/dev/z/abz.ts", + "/dev/z/aba.ts", + "/dev/z/b.ts", + "/dev/z/bbz.ts", + "/dev/z/bba.ts", + "/dev/x/a.ts", + "/dev/x/b.ts", + "/dev/x/y/a.ts", + "/dev/x/y/b.ts", + "/dev/js/a.js", + "/dev/js/b.js", ]); - const caseInsensitiveMixedExtensionHost = createMockParseConfigHost(/*ignoreCase*/ true, caseInsensitiveBasePath, [ - "a.ts", - "a.d.ts", - "a.js", - "b.tsx", - "b.d.ts", - "b.jsx", - "c.tsx", - "c.js", - "d.js", - "e.jsx", - "f.other" + const caseInsensitiveMixedExtensionHost = new MockParseConfigHost(caseInsensitiveBasePath, /*useCaseSensitiveFileNames*/ false, [ + "c:/dev/a.ts", + "c:/dev/a.d.ts", + "c:/dev/a.js", + "c:/dev/b.tsx", + "c:/dev/b.d.ts", + "c:/dev/b.jsx", + "c:/dev/c.tsx", + "c:/dev/c.js", + "c:/dev/d.js", + "c:/dev/e.jsx", + "c:/dev/f.other" ]); describe("expandFiles", () => { @@ -226,7 +252,7 @@ namespace ts { const expected: ts.ExpandResult = { fileNames: [], wildcardDirectories: { - "c:/dev": ts.WatchDirectoryFlags.None + "c:/dev": ts.WatchDirectoryFlags.Recursive }, }; const actual = ts.expandFiles(/*fileNames*/ undefined, includeSpecs, /*excludeSpecs*/ undefined, caseInsensitiveBasePath, {}, caseInsensitiveHost); @@ -239,7 +265,7 @@ namespace ts { const expected: ts.ExpandResult = { fileNames: ["c:/dev/a.ts"], wildcardDirectories: { - "c:/dev": ts.WatchDirectoryFlags.None + "c:/dev": ts.WatchDirectoryFlags.Recursive }, }; const actual = ts.expandFiles(fileNames, includeSpecs, excludeSpecs, caseInsensitiveBasePath, {}, caseInsensitiveHost); @@ -283,6 +309,23 @@ namespace ts { const actual = ts.expandFiles(/*fileNames*/ undefined, includeSpecs, /*excludeSpecs*/ undefined, caseInsensitiveBasePath, { allowJs: true }, caseInsensitiveHost); assert.deepEqual(actual, expected); }); + it("include paths outside of the project", () => { + const includeSpecs = ["*", "c:/ext/*"]; + const expected: ts.ExpandResult = { + fileNames: [ + "c:/dev/a.ts", + "c:/dev/b.ts", + "c:/dev/c.d.ts", + "c:/ext/ext.ts", + ], + wildcardDirectories: { + "c:/dev": ts.WatchDirectoryFlags.None, + "c:/ext": ts.WatchDirectoryFlags.None + } + }; + const actual = ts.expandFiles(/*fileNames*/ undefined, includeSpecs, /*excludeSpecs*/ undefined, caseInsensitiveBasePath, {}, caseInsensitiveHost); + assert.deepEqual(actual, expected); + }); }); describe("when called from parseJsonConfigFileContent", () => { @@ -376,110 +419,4 @@ namespace ts { }); }); }); - - interface DirectoryEntry { - files: ts.Map; - directories: ts.Map; - } - - interface TestParseConfigHost extends ts.ParseConfigHost { - basePath: string; - } - - function createMockParseConfigHost(ignoreCase: boolean, basePath: string, files: string[]): TestParseConfigHost { - const fileSet: ts.Map = {}; - const directorySet: ts.Map = {}; - const emptyDirectory: DirectoryEntry = { files: {}, directories: {} }; - - files.sort((a, b) => ts.comparePaths(a, b, basePath, ignoreCase)); - for (const file of files) { - addFile(ts.getNormalizedAbsolutePath(file, basePath)); - } - - return { - useCaseSensitiveFileNames: !ignoreCase, - basePath, - fileExists, - directoryExists, - readDirectory, - readFileNames, - readDirectoryNames - }; - - function fileExists(path: string): boolean { - path = ts.getNormalizedAbsolutePath(path, basePath); - path = ts.removeTrailingDirectorySeparator(path); - const fileKey = ignoreCase ? path.toLowerCase() : path; - return ts.hasProperty(fileSet, fileKey); - } - - function directoryExists(path: string): boolean { - path = ts.getNormalizedAbsolutePath(path, basePath); - path = ts.removeTrailingDirectorySeparator(path); - const directoryKey = ignoreCase ? path.toLowerCase() : path; - return ts.hasProperty(directorySet, directoryKey); - } - - function readDirectory(rootDir: string, extension?: string, exclude?: string[]): string[] { - throw new Error("Not implemented"); - } - - function readFileNames(path: string) { - const { files } = getDirectoryEntry(path) || emptyDirectory; - const result: string[] = []; - ts.forEachKey(files, key => { result.push(key); }); - result.sort((a, b) => ts.compareStrings(a, b, ignoreCase)); - return result; - } - - function readDirectoryNames(path: string) { - const { directories } = getDirectoryEntry(path); // || emptyDirectory; - const result: string[] = []; - ts.forEachKey(directories, key => { result.push(key); }); - result.sort((a, b) => ts.compareStrings(a, b, ignoreCase)); - return result; - } - - function getDirectoryEntry(path: string) { - path = ts.getNormalizedAbsolutePath(path, basePath); - path = ts.removeTrailingDirectorySeparator(path); - const directoryKey = ignoreCase ? path.toLowerCase() : path; - return ts.getProperty(directorySet, directoryKey); - } - - function addFile(file: string) { - const fileKey = ignoreCase ? file.toLowerCase() : file; - if (!ts.hasProperty(fileSet, fileKey)) { - fileSet[fileKey] = file; - const name = ts.getBaseFileName(file); - const parent = ts.getDirectoryPath(file); - addToDirectory(parent, name, "file"); - } - } - - function addDirectory(directory: string) { - directory = ts.removeTrailingDirectorySeparator(directory); - const directoryKey = ignoreCase ? directory.toLowerCase() : directory; - if (!ts.hasProperty(directorySet, directoryKey)) { - directorySet[directoryKey] = { files: {}, directories: {} }; - const name = ts.getBaseFileName(directory); - const parent = ts.getDirectoryPath(directory); - if (parent !== directory) { - addToDirectory(parent, name, "directory"); - } - } - } - - function addToDirectory(directory: string, entry: string, type: "file" | "directory") { - addDirectory(directory); - directory = ts.removeTrailingDirectorySeparator(directory); - const directoryKey = ignoreCase ? directory.toLowerCase() : directory; - const entryKey = ignoreCase ? entry.toLowerCase() : entry; - const directoryEntry = directorySet[directoryKey]; - const entries = type === "file" ? directoryEntry.files : directoryEntry.directories; - if (!ts.hasProperty(entries, entryKey)) { - entries[entryKey] = entry; - } - } - } } \ No newline at end of file diff --git a/tests/cases/unittests/session.ts b/tests/cases/unittests/session.ts index 8e35baa3910..7c44b895ce0 100644 --- a/tests/cases/unittests/session.ts +++ b/tests/cases/unittests/session.ts @@ -16,8 +16,6 @@ namespace ts.server { getExecutingFilePath(): string { return void 0; }, getCurrentDirectory(): string { return void 0; }, readDirectory(): string[] { return []; }, - readDirectoryNames(): string[] { return []; }, - readFileNames(): string[] { return []; }, exit(): void {} }; const mockLogger: Logger = {