microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2026-05-15 21:36:50 -05:00
Code Issues Packages Projects Releases 212 Wiki Activity

Limit the resolution invalidation to max number of files as invalidation for larger cache might take more time than to just recalculate resolutions

Browse Source
This commit is contained in:
Sheetal Nandi
2017-09-07 17:27:00 -07:00
parent b179cd1e1c
commit 67f9533c67
6 changed files with 212 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

199
src/compiler/resolutionCache.ts
View File

@@ -12,6 +12,7 @@ namespace ts {
resolveTypeReferenceDirectives(typeDirectiveNames: string[], containingFile: string): ResolvedTypeReferenceDirective[];
invalidateResolutionOfFile(filePath: Path): void;
removeResolutionsOfFile(filePath: Path): void;
createHasInvalidatedResolution(): HasInvalidatedResolution;
startCachingPerDirectoryResolution(): void;
@@ -25,7 +26,7 @@ namespace ts {
clear(): void;
}
interface NameResolutionWithFailedLookupLocations {
interface ResolutionWithFailedLookupLocations {
readonly failedLookupLocations: ReadonlyArray<string>;
isInvalidated?: boolean;
}
@@ -45,6 +46,7 @@ namespace ts {
projectName?: string;
getGlobalCache?(): string | undefined;
writeLog(s: string): void;
maxNumberOfFilesToIterateForInvalidation?: number;
}
interface DirectoryWatchesOfFailedLookup {
@@ -54,9 +56,17 @@ namespace ts {
refCount: number;
}
/*@internal*/
export const maxNumberOfFilesToIterateForInvalidation = 256;
interface GetResolutionWithResolvedFileName<T extends ResolutionWithFailedLookupLocations = ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName = ResolutionWithResolvedFileName> {
(resolution: T): R;
}
export function createResolutionCache(resolutionHost: ResolutionCacheHost): ResolutionCache {
let filesWithChangedSetOfUnresolvedImports: Path[] | undefined;
let filesWithInvalidatedResolutions: Map<true> | undefined;
let allFilesHaveInvalidatedResolution = false;
// The resolvedModuleNames and resolvedTypeReferenceDirectives are the cache of resolutions per file.
// The key in the map is source file's path.
@@ -86,6 +96,7 @@ namespace ts {
finishCachingPerDirectoryResolution,
resolveModuleNames,
resolveTypeReferenceDirectives,
removeResolutionsOfFile,
invalidateResolutionOfFile,
createHasInvalidatedResolution,
setRootDirectory,
@@ -121,6 +132,7 @@ namespace ts {
closeTypeRootsWatch();
resolvedModuleNames.clear();
resolvedTypeReferenceDirectives.clear();
allFilesHaveInvalidatedResolution = false;
Debug.assert(perDirectoryResolvedModuleNames.size === 0 && perDirectoryResolvedTypeReferenceDirectives.size === 0);
}
@@ -135,6 +147,11 @@ namespace ts {
}
function createHasInvalidatedResolution(): HasInvalidatedResolution {
if (allFilesHaveInvalidatedResolution) {
// Any file asked would have invalidated resolution
filesWithInvalidatedResolutions = undefined;
return returnTrue;
}
const collected = filesWithInvalidatedResolutions;
filesWithInvalidatedResolutions = undefined;
return path => collected && collected.has(path);
@@ -145,6 +162,7 @@ namespace ts {
}
function finishCachingPerDirectoryResolution() {
allFilesHaveInvalidatedResolution = false;
directoryWatchesOfFailedLookups.forEach((watcher, path) => {
if (watcher.refCount === 0) {
directoryWatchesOfFailedLookups.delete(path);
@@ -178,13 +196,13 @@ namespace ts {
return primaryResult;
}
function resolveNamesWithLocalCache<T extends NameResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName>(
function resolveNamesWithLocalCache<T extends ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName>(
names: string[],
containingFile: string,
cache: Map<Map<T>>,
perDirectoryCache: Map<Map<T>>,
loader: (name: string, containingFile: string, options: CompilerOptions, host: ModuleResolutionHost) => T,
getResolutionFromNameResolutionWithFailedLookupLocations: (s: T) => R,
getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>,
logChanges: boolean): R[] {
const path = resolutionHost.toPath(containingFile);
@@ -203,7 +221,7 @@ namespace ts {
for (const name of names) {
let resolution = resolutionsInFile.get(name);
// Resolution is valid if it is present and not invalidated
if (!resolution || resolution.isInvalidated) {
if (allFilesHaveInvalidatedResolution || !resolution || resolution.isInvalidated) {
const existingResolution = resolution;
const resolutionInDirectory = perDirectoryResolution.get(name);
if (resolutionInDirectory) {
@@ -225,7 +243,7 @@ namespace ts {
}
Debug.assert(resolution !== undefined && !resolution.isInvalidated);
seenNamesInFile.set(name, true);
resolvedModules.push(getResolutionFromNameResolutionWithFailedLookupLocations(resolution));
resolvedModules.push(getResolutionWithResolvedFileName(resolution));
}
// Stop watching and remove the unused name
@@ -245,8 +263,8 @@ namespace ts {
if (!oldResolution || !newResolution || oldResolution.isInvalidated) {
return false;
}
const oldResult = getResolutionFromNameResolutionWithFailedLookupLocations(oldResolution);
const newResult = getResolutionFromNameResolutionWithFailedLookupLocations(newResolution);
const oldResult = getResolutionWithResolvedFileName(oldResolution);
const newResult = getResolutionWithResolvedFileName(newResolution);
if (oldResult === newResult) {
return true;
}
@@ -321,9 +339,7 @@ namespace ts {
return fileExtensionIsOneOf(path, failedLookupDefaultExtensions);
}
function watchFailedLookupLocationOfResolution<T extends NameResolutionWithFailedLookupLocations>(
resolution: T, startIndex?: number
) {
function watchFailedLookupLocationOfResolution(resolution: ResolutionWithFailedLookupLocations, startIndex?: number) {
if (resolution && resolution.failedLookupLocations) {
for (let i = startIndex || 0; i < resolution.failedLookupLocations.length; i++) {
const failedLookupLocation = resolution.failedLookupLocations[i];
@@ -346,15 +362,11 @@ namespace ts {
}
}
function stopWatchFailedLookupLocationOfResolution<T extends NameResolutionWithFailedLookupLocations>(
resolution: T
) {
function stopWatchFailedLookupLocationOfResolution(resolution: ResolutionWithFailedLookupLocations) {
stopWatchFailedLookupLocationOfResolutionFrom(resolution, 0);
}
function stopWatchFailedLookupLocationOfResolutionFrom<T extends NameResolutionWithFailedLookupLocations>(
resolution: T, startIndex: number
) {
function stopWatchFailedLookupLocationOfResolutionFrom(resolution: ResolutionWithFailedLookupLocations, startIndex: number) {
if (resolution && resolution.failedLookupLocations) {
for (let i = startIndex; i < resolution.failedLookupLocations.length; i++) {
const failedLookupLocation = resolution.failedLookupLocations[i];
@@ -387,107 +399,106 @@ namespace ts {
// If the files are added to project root or node_modules directory, always run through the invalidation process
// Otherwise run through invalidation only if adding to the immediate directory
if (dirPath === rootPath || isNodeModulesDirectory(dirPath) || getDirectoryPath(fileOrFolderPath) === dirPath) {
let isChangedFailedLookupLocation: (location: string) => boolean;
if (dirPath === fileOrFolderPath) {
// Watching directory is created
// Invalidate any resolution has failed lookup in this directory
isChangedFailedLookupLocation = location => isInDirectoryPath(dirPath, resolutionHost.toPath(location));
}
else {
// Some file or folder in the watching directory is created
// Return early if it does not have any of the watching extension or not the custom failed lookup path
if (!isPathWithDefaultFailedLookupExtension(fileOrFolderPath) && !customFailedLookupPaths.has(fileOrFolderPath)) {
return;
}
// Resolution need to be invalidated if failed lookup location is same as the file or folder getting created
isChangedFailedLookupLocation = location => resolutionHost.toPath(location) === fileOrFolderPath;
}
const hasChangedFailedLookupLocation = (resolution: NameResolutionWithFailedLookupLocations) => some(resolution.failedLookupLocations, isChangedFailedLookupLocation);
if (invalidateResolutionOfFailedLookupLocation(hasChangedFailedLookupLocation)) {
if (invalidateResolutionOfFailedLookupLocation(fileOrFolderPath, dirPath === fileOrFolderPath)) {
resolutionHost.onInvalidatedResolution();
}
}
}, WatchDirectoryFlags.Recursive);
}
function invalidateResolutionCache<T extends NameResolutionWithFailedLookupLocations>(
function removeResolutionsOfFileFromCache(cache: Map<Map<ResolutionWithFailedLookupLocations>>, filePath: Path) {
// Deleted file, stop watching failed lookups for all the resolutions in the file
const resolutions = cache.get(filePath);
if (resolutions) {
resolutions.forEach(stopWatchFailedLookupLocationOfResolution);
cache.delete(filePath);
}
}
function removeResolutionsOfFile(filePath: Path) {
removeResolutionsOfFileFromCache(resolvedModuleNames, filePath);
removeResolutionsOfFileFromCache(resolvedTypeReferenceDirectives, filePath);
}
function invalidateResolutionCache<T extends ResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName>(
cache: Map<Map<T>>,
ignoreFile: (resolutions: Map<T>, containingFilePath: Path) => boolean,
isInvalidatedResolution: (resolution: T) => boolean
isInvalidatedResolution: (resolution: T, getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>) => boolean,
getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName<T, R>
) {
const seen = createMap<Map<true>>();
cache.forEach((resolutions, containingFilePath) => {
if (!ignoreFile(resolutions, containingFilePath as Path) && resolutions) {
const dirPath = getDirectoryPath(containingFilePath);
let seenInDir = seen.get(dirPath);
if (!seenInDir) {
seenInDir = createMap<true>();
seen.set(dirPath, seenInDir);
}
resolutions.forEach((resolution, name) => {
if (seenInDir.has(name)) {
return;
}
seenInDir.set(name, true);
if (!resolution.isInvalidated && isInvalidatedResolution(resolution)) {
// Mark the file as needing re-evaluation of module resolution instead of using it blindly.
resolution.isInvalidated = true;
(filesWithInvalidatedResolutions || (filesWithInvalidatedResolutions = createMap<true>())).set(containingFilePath, true);
}
});
const dirPath = getDirectoryPath(containingFilePath);
let seenInDir = seen.get(dirPath);
if (!seenInDir) {
seenInDir = createMap<true>();
seen.set(dirPath, seenInDir);
}
resolutions.forEach((resolution, name) => {
if (seenInDir.has(name)) {
return;
}
seenInDir.set(name, true);
if (!resolution.isInvalidated && isInvalidatedResolution(resolution, getResolutionWithResolvedFileName)) {
// Mark the file as needing re-evaluation of module resolution instead of using it blindly.
resolution.isInvalidated = true;
(filesWithInvalidatedResolutions || (filesWithInvalidatedResolutions = createMap<true>())).set(containingFilePath, true);
}
});
});
}
function invalidateResolutionCacheOfDeletedFile<T extends NameResolutionWithFailedLookupLocations, R extends ResolutionWithResolvedFileName>(
deletedFilePath: Path,
cache: Map<Map<T>>,
getResolutionFromNameResolutionWithFailedLookupLocations: (s: T) => R,
) {
invalidateResolutionCache(
cache,
// Ignore file thats same as deleted file path, and handle it here
(resolutions, containingFilePath) => {
if (containingFilePath !== deletedFilePath) {
return false;
}
function hasReachedResolutionIterationLimit() {
const maxSize = resolutionHost.maxNumberOfFilesToIterateForInvalidation || maxNumberOfFilesToIterateForInvalidation;
return resolvedModuleNames.size > maxSize || resolvedTypeReferenceDirectives.size > maxSize;
}
// Deleted file, stop watching failed lookups for all the resolutions in the file
cache.delete(containingFilePath);
resolutions.forEach(stopWatchFailedLookupLocationOfResolution);
return true;
},
function invalidateResolutions(
isInvalidatedResolution: (resolution: ResolutionWithFailedLookupLocations, getResolutionWithResolvedFileName: GetResolutionWithResolvedFileName) => boolean,
) {
// If more than maxNumberOfFilesToIterateForInvalidation present,
// just invalidated all files and recalculate the resolutions for files instead
if (hasReachedResolutionIterationLimit()) {
allFilesHaveInvalidatedResolution = true;
return;
}
invalidateResolutionCache(resolvedModuleNames, isInvalidatedResolution, getResolvedModule);
invalidateResolutionCache(resolvedTypeReferenceDirectives, isInvalidatedResolution, getResolvedTypeReferenceDirective);
}
function invalidateResolutionOfFile(filePath: Path) {
removeResolutionsOfFile(filePath);
invalidateResolutions(
// Resolution is invalidated if the resulting file name is same as the deleted file path
resolution => {
const result = getResolutionFromNameResolutionWithFailedLookupLocations(resolution);
return result && resolutionHost.toPath(result.resolvedFileName) === deletedFilePath;
(resolution, getResolutionWithResolvedFileName) => {
const result = getResolutionWithResolvedFileName(resolution);
return result && resolutionHost.toPath(result.resolvedFileName) === filePath;
}
);
}
function invalidateResolutionOfFile(filePath: Path) {
invalidateResolutionCacheOfDeletedFile(filePath, resolvedModuleNames, getResolvedModule);
invalidateResolutionCacheOfDeletedFile(filePath, resolvedTypeReferenceDirectives, getResolvedTypeReferenceDirective);
}
function invalidateResolutionCacheOfFailedLookupLocation<T extends NameResolutionWithFailedLookupLocations>(
cache: Map<Map<T>>,
hasChangedFailedLookupLocation: (resolution: T) => boolean
) {
invalidateResolutionCache(
cache,
// Do not ignore any file
returnFalse,
function invalidateResolutionOfFailedLookupLocation(fileOrFolderPath: Path, isCreatingWatchedDirectory: boolean) {
let isChangedFailedLookupLocation: (location: string) => boolean;
if (isCreatingWatchedDirectory) {
// Watching directory is created
// Invalidate any resolution has failed lookup in this directory
isChangedFailedLookupLocation = location => isInDirectoryPath(fileOrFolderPath, resolutionHost.toPath(location));
}
else {
// Some file or folder in the watching directory is created
// Return early if it does not have any of the watching extension or not the custom failed lookup path
if (!isPathWithDefaultFailedLookupExtension(fileOrFolderPath) && !customFailedLookupPaths.has(fileOrFolderPath)) {
return false;
}
// Resolution need to be invalidated if failed lookup location is same as the file or folder getting created
isChangedFailedLookupLocation = location => resolutionHost.toPath(location) === fileOrFolderPath;
}
const hasChangedFailedLookupLocation = (resolution: ResolutionWithFailedLookupLocations) => some(resolution.failedLookupLocations, isChangedFailedLookupLocation);
const invalidatedFilesCount = filesWithInvalidatedResolutions && filesWithInvalidatedResolutions.size;
invalidateResolutions(
// Resolution is invalidated if the resulting file name is same as the deleted file path
hasChangedFailedLookupLocation
);
}
function invalidateResolutionOfFailedLookupLocation(hasChangedFailedLookupLocation: (resolution: NameResolutionWithFailedLookupLocations) => boolean) {
const invalidatedFilesCount = filesWithInvalidatedResolutions && filesWithInvalidatedResolutions.size;
invalidateResolutionCacheOfFailedLookupLocation(resolvedModuleNames, hasChangedFailedLookupLocation);
invalidateResolutionCacheOfFailedLookupLocation(resolvedTypeReferenceDirectives, hasChangedFailedLookupLocation);
return filesWithInvalidatedResolutions && filesWithInvalidatedResolutions.size !== invalidatedFilesCount;
return allFilesHaveInvalidatedResolution || filesWithInvalidatedResolutions && filesWithInvalidatedResolutions.size !== invalidatedFilesCount;
}
function closeTypeRootsWatch() {

2
src/compiler/watchedProgram.ts
View File

@@ -501,7 +501,7 @@ namespace ts {
}
else if (hostSourceFileInfo.sourceFile === oldSourceFile) {
sourceFilesCache.delete(oldSourceFile.path);
resolutionCache.invalidateResolutionOfFile(oldSourceFile.path);
resolutionCache.removeResolutionsOfFile(oldSourceFile.path);
}
}
}

79
src/harness/unittests/tsserverProjectSystem.ts
View File

@@ -5168,6 +5168,85 @@ namespace ts.projectSystem {
verifyProjectChangedEvent([referenceFile1, moduleFile2], [libFile, moduleFile2, referenceFile1, configFile]);
});
});
describe("resolution when resolution cache size", () => {
function verifyWithMaxCacheLimit(limitHit: boolean) {
const file1: FileOrFolder = {
path: "/a/b/project/file1.ts",
content: 'import a from "file2"'
};
const file2: FileOrFolder = {
path: "/a/b/node_modules/file2.d.ts",
content: "export class a { }"
};
const file3: FileOrFolder = {
path: "/a/b/project/file3.ts",
content: "export class c { }"
};
const configFile: FileOrFolder = {
path: "/a/b/project/tsconfig.json",
content: JSON.stringify({ compilerOptions: { typeRoots: [] } })
};
const projectFiles = [file1, file3, libFile, configFile];
const watchedRecursiveDirectories = ["/a/b/project", "/a/b/node_modules", "/a/node_modules", "/node_modules"];
const host = createServerHost(projectFiles);
const { session, verifyInitialOpen, verifyProjectChangedEventHandler } = createSession(host);
const projectService = session.getProjectService();
verifyInitialOpen(file1);
checkNumberOfProjects(projectService, { configuredProjects: 1 });
const project = projectService.configuredProjects.get(configFile.path);
verifyProject();
if (limitHit) {
(project as ResolutionCacheHost).maxNumberOfFilesToIterateForInvalidation = 1;
}
file3.content += "export class d {}";
host.reloadFS(projectFiles);
host.checkTimeoutQueueLengthAndRun(2);
// Since this is first event
verifyProject();
verifyProjectChangedEventHandler([{
eventName: server.ProjectChangedEvent,
data: {
project,
changedFiles: [libFile.path, file1.path, file3.path],
filesToEmit: [file1.path, file3.path]
}
}]);
projectFiles.push(file2);
host.reloadFS(projectFiles);
host.runQueuedTimeoutCallbacks();
watchedRecursiveDirectories.length = 2;
verifyProject();
const changedFiles = limitHit ? [file1.path, file2.path, file3.path, libFile.path] : [file1.path, file2.path];
verifyProjectChangedEventHandler([{
eventName: server.ProjectChangedEvent,
data: {
project,
changedFiles,
filesToEmit: changedFiles
}
}]);
function verifyProject() {
checkProjectActualFiles(project, map(projectFiles, file => file.path));
checkWatchedDirectories(host, [], /*recursive*/ false);
checkWatchedDirectories(host, watchedRecursiveDirectories, /*recursive*/ true);
}
}
it("limit not hit", () => {
verifyWithMaxCacheLimit(/*limitHit*/ false);
});
it("limit hit", () => {
verifyWithMaxCacheLimit(/*limitHit*/ true);
});
});
}
describe("when event handler is set in the session", () => {

12
src/server/editorServices.ts
View File

@@ -876,6 +876,8 @@ namespace ts.server {
unorderedRemoveItem(this.openFiles, info);
const fileExists = this.host.fileExists(info.fileName);
// collect all projects that should be removed
let projectsToRemove: Project[];
for (const p of info.containingProjects) {
@@ -896,7 +898,7 @@ namespace ts.server {
(projectsToRemove || (projectsToRemove = [])).push(p);
}
else {
p.removeFile(info);
p.removeFile(info, fileExists, /*detachFromProject*/ true);
}
}
@@ -926,7 +928,7 @@ namespace ts.server {
// If the current info is being just closed - add the watcher file to track changes
// But if file was deleted, handle that part
if (this.host.fileExists(info.fileName)) {
if (fileExists) {
this.watchClosedScriptInfo(info);
}
else {
@@ -1447,7 +1449,7 @@ namespace ts.server {
path = normalizedPathToPath(normalizedPath, this.currentDirectory, this.toCanonicalFileName);
const existingValue = projectRootFilesMap.get(path);
if (isScriptInfo(existingValue)) {
project.removeFile(existingValue);
project.removeFile(existingValue, /*fileExists*/ false, /*detachFromProject*/ true);
}
projectRootFilesMap.set(path, normalizedPath);
scriptInfo = normalizedPath;
@@ -1476,7 +1478,7 @@ namespace ts.server {
projectRootFilesMap.forEach((value, path) => {
if (!newRootScriptInfoMap.has(path)) {
if (isScriptInfo(value)) {
project.removeFile(value);
project.removeFile(value, project.fileExists(path), /*detachFromProject*/ true);
}
else {
projectRootFilesMap.delete(path);
@@ -1806,7 +1808,7 @@ namespace ts.server {
this.removeProject(inferredProject);
}
else {
inferredProject.removeFile(info);
inferredProject.removeFile(info, /*fileExists*/ true, /*detachFromProject*/ true);
}
}
}

31
src/server/project.ts
View File

@@ -667,11 +667,17 @@ namespace ts.server {
this.markAsDirty();
}
removeFile(info: ScriptInfo, detachFromProject = true) {
removeFile(info: ScriptInfo, fileExists: boolean, detachFromProject: boolean) {
if (this.isRoot(info)) {
this.removeRoot(info);
}
this.resolutionCache.invalidateResolutionOfFile(info.path);
if (fileExists) {
// If file is present, just remove the resolutions for the file
this.resolutionCache.removeResolutionsOfFile(info.path);
}
else {
this.resolutionCache.invalidateResolutionOfFile(info.path);
}
this.cachedUnresolvedImportsPerFile.remove(info.path);
if (detachFromProject) {
@@ -807,10 +813,7 @@ namespace ts.server {
continue;
}
// new program does not contain this file - detach it from the project
const scriptInfoToDetach = this.projectService.getScriptInfo(f.fileName);
if (scriptInfoToDetach) {
scriptInfoToDetach.detachFromProject(this);
}
this.detachScriptInfoFromProject(f.fileName);
}
}
@@ -838,17 +841,21 @@ namespace ts.server {
const scriptInfo = this.projectService.getOrCreateScriptInfoNotOpenedByClient(inserted, this.partialSystem);
scriptInfo.attachToProject(this);
},
removed => {
const scriptInfoToDetach = this.projectService.getScriptInfo(removed);
if (scriptInfoToDetach) {
scriptInfoToDetach.detachFromProject(this);
}
});
removed => this.detachScriptInfoFromProject(removed)
);
const elapsed = timestamp() - start;
this.writeLog(`Finishing updateGraphWorker: Project: ${this.getProjectName()} structureChanged: ${hasChanges} Elapsed: ${elapsed}ms`);
return hasChanges;
}
private detachScriptInfoFromProject(uncheckedFileName: string) {
const scriptInfoToDetach = this.projectService.getScriptInfo(uncheckedFileName);
if (scriptInfoToDetach) {
scriptInfoToDetach.detachFromProject(this);
this.resolutionCache.removeResolutionsOfFile(scriptInfoToDetach.path);
}
}
private addMissingFileWatcher(missingFilePath: Path) {
const fileWatcher = this.projectService.watchFile(
this.projectService.host,

2
src/server/scriptInfo.ts
View File

@@ -282,7 +282,7 @@ namespace ts.server {
}
const isInfoRoot = p.isRoot(this);
// detach is unnecessary since we'll clean the list of containing projects anyways
p.removeFile(this, /*detachFromProjects*/ false);
p.removeFile(this, /*fileExists*/ false, /*detachFromProjects*/ false);
// If the info was for the external or configured project's root,
// add missing file as the root
if (isInfoRoot && p.projectKind !== ProjectKind.Inferred) {