diff --git a/src/server/project.ts b/src/server/project.ts
index ca9cb2a68da..7546ae9477f 100644
--- a/src/server/project.ts
+++ b/src/server/project.ts
@@ -1194,7 +1194,6 @@ export abstract class Project implements LanguageServiceHost, ModuleResolutionHo
         if (!fileExists) {
             this.resolutionCache.invalidateResolutionOfFile(info.path);
         }
-        this.cachedUnresolvedImportsPerFile.delete(info.path);
 
         if (detachFromProject) {
             info.detachFromProject(this);
@@ -2082,6 +2081,12 @@ function getUnresolvedImports(program: Program, cachedUnresolvedImportsPerFile:
     const ambientModules = program.getTypeChecker().getAmbientModules().map(mod => stripQuotes(mod.getName()));
     const result = sortAndDeduplicate(flatMap(sourceFiles, sourceFile =>
         extractUnresolvedImportsFromSourceFile(sourceFile, ambientModules, cachedUnresolvedImportsPerFile)));
+    // Remove files from the cache if they arent in program
+    if (cachedUnresolvedImportsPerFile.size !== program.getSourceFiles().length) {
+        cachedUnresolvedImportsPerFile.forEach((_value, key) => {
+            if (!program.getSourceFileByPath(key)) cachedUnresolvedImportsPerFile.delete(key);
+        });
+    }
     tracing?.pop();
     return result;
 }