[WIP] typings discovery in tsserver

Jakefile.js

@@ -186,6 +186,7 @@ var harnessSources = harnessCoreSources.concat([
     "scriptInfo.ts",
     "lsHost.ts",
     "project.ts",
+    "typingsCache.ts",
     "editorServices.ts",
     "protocol.d.ts",
     "session.ts",

src/server/editorServices.ts

@@ -6,6 +6,7 @@
 /// <reference path="scriptVersionCache.ts"/>
 /// <reference path="lsHost.ts"/>
 /// <reference path="project.ts"/>
+/// <reference path="typingsCache.ts"/>
 
 namespace ts.server {
     export const maxProgramSizeForNonTsFiles = 20 * 1024 * 1024;
@@ -130,6 +131,9 @@ namespace ts.server {
     }
 
     export class ProjectService {
+
+        public readonly typingsCache: TypingsCache;
+
         private readonly documentRegistry: DocumentRegistry;
 
         /**
@@ -654,6 +658,7 @@ namespace ts.server {
                 compilerOptions: parsedCommandLine.options,
                 configHasFilesProperty: configObj.config["files"] !== undefined,
                 wildcardDirectories: parsedCommandLine.wildcardDirectories,
+                typingOptions: parsedCommandLine.typingOptions
             };
             return { success: true, projectOptions };
         }
@@ -697,6 +702,7 @@ namespace ts.server {
                 this.documentRegistry,
                 projectOptions.configHasFilesProperty,
                 projectOptions.compilerOptions,
+                projectOptions.typingOptions,
                 projectOptions.wildcardDirectories,
                 /*languageServiceEnabled*/ !sizeLimitExceeded);
 

src/server/lsHost.ts

@@ -101,7 +101,7 @@ namespace ts.server {
         }
 
         getScriptFileNames() {
-            return this.project.getRootFiles();
+            return this.project.getRootFilesLSHost();
         }
 
         getScriptKind(fileName: string) {

src/server/project.ts

@@ -2,6 +2,7 @@
 /// <reference path="utilities.ts"/>
 /// <reference path="scriptInfo.ts"/>
 /// <reference path="lsHost.ts"/>
+/// <reference path="typingsCache.ts"/>
 
 namespace ts.server {
 
@@ -25,7 +26,6 @@ namespace ts.server {
         private program: ts.Program;
 
         private languageService: LanguageService;
-
         /**
          * Set of files that was returned from the last call to getChangesSinceVersion.
          */
@@ -47,6 +47,8 @@ namespace ts.server {
          */
         private projectStateVersion = 0;
 
+        private typingFiles: string[];
+
         constructor(
             readonly projectKind: ProjectKind,
             readonly projectService: ProjectService,
@@ -74,7 +76,7 @@ namespace ts.server {
             this.markAsDirty();
         }
 
-        getLanguageService(ensureSynchronized = true): LanguageService  {
+        getLanguageService(ensureSynchronized = true): LanguageService {
             if (ensureSynchronized) {
                 this.updateGraph();
             }
@@ -136,6 +138,21 @@ namespace ts.server {
             return this.rootFiles && this.rootFiles.map(info => info.fileName);
         }
 
+        getRootFilesLSHost() {
+            const result: string[] = [];
+            if (this.rootFiles) {
+                for (const f of this.rootFiles) {
+                    result.push(f.fileName);
+                }
+                if (this.typingFiles) {
+                    for (const f of this.typingFiles) {
+                        result.push(f);
+                    }
+                }
+            }
+            return result;
+        }
+
         getRootScriptInfos() {
             return this.rootFiles;
         }
@@ -209,16 +226,35 @@ namespace ts.server {
             if (!this.languageServiceEnabled) {
                 return true;
             }
+            const hasChanges = this.updateGraphWorker();
+            if (hasChanges) {
+                if (this.setTypings(this.projectService.typingsCache.getTypingsForProject(this))) {
+                    this.updateGraphWorker();
+                }
+                this.projectStructureVersion++;
+            }
+            return hasChanges;
+        }
 
+        setTypings(typings: string[]): boolean {
+            if (typings === this.typingFiles) {
+                return false;
+            }
+            this.typingFiles = typings;
+            this.markAsDirty();
+            return true;
+        }
+
+        private updateGraphWorker() {
             const oldProgram = this.program;
             this.program = this.languageService.getProgram();
 
-            const oldProjectStructureVersion = this.projectStructureVersion;
+            let hasChanges = false;
             // bump up the version if
             // - oldProgram is not set - this is a first time updateGraph is called
             // - newProgram is different from the old program and structure of the old program was not reused.
             if (!oldProgram || (this.program !== oldProgram && !oldProgram.structureIsReused)) {
-                this.projectStructureVersion++;
+                hasChanges = true;
                 if (oldProgram) {
                     for (const f of oldProgram.getSourceFiles()) {
                         if (this.program.getSourceFileByPath(f.path)) {
@@ -232,8 +268,7 @@ namespace ts.server {
                     }
                 }
             }
-
-            return oldProjectStructureVersion === this.projectStructureVersion;
+            return hasChanges;
         }
 
         getScriptInfoLSHost(fileName: string) {
@@ -392,11 +427,16 @@ namespace ts.server {
             documentRegistry: ts.DocumentRegistry,
             hasExplicitListOfFiles: boolean,
             compilerOptions: CompilerOptions,
+            private typingOptions: TypingOptions,
             private wildcardDirectories: Map<WatchDirectoryFlags>,
             languageServiceEnabled: boolean) {
             super(ProjectKind.Configured, projectService, documentRegistry, hasExplicitListOfFiles, languageServiceEnabled, compilerOptions);
         }
 
+        getTypingOptions() {
+            return this.typingOptions;
+        }
+
         getProjectName() {
             return this.configFileName;
         }

src/server/session.ts

@@ -147,6 +147,7 @@ namespace ts.server {
     }
 
     export class Session {
+        private readonly gcTimer: GcTimer;
         protected projectService: ProjectService;
         private errorTimer: any; /*NodeJS.Timer | number*/
         private immediateId: any;
@@ -165,6 +166,7 @@ namespace ts.server {
                 new ProjectService(host, logger, cancellationToken, useSingleInferredProject, (eventName, project, fileName) => {
                     this.handleEvent(eventName, project, fileName);
                 });
+            this.gcTimer = new GcTimer(host, /*delay*/ 15000, logger);
         }
 
         private handleEvent(eventName: string, project: Project, fileName: NormalizedPath) {
@@ -1464,6 +1466,7 @@ namespace ts.server {
         }
 
         public onMessage(message: string) {
+            this.gcTimer.scheduleCollect();
             let start: number[];
             if (this.logger.hasLevel(LogLevel.requestTime)) {
                 start = this.hrtime();

src/server/tsconfig.json

@@ -15,6 +15,11 @@
     "files": [
         "../services/shims.ts",
         "../services/utilities.ts",
+        "utilities.ts",
+        "scriptVersionCache.ts",
+        "scriptInfo.ts",
+        "lshost.ts",
+        "project.ts",
         "editorServices.ts",
         "protocol.d.ts",
         "session.ts",

src/server/tsconfig.library.json

@@ -10,6 +10,13 @@
         "types": []
     },
     "files": [
+        "../services/shims.ts",
+        "../services/utilities.ts",
+        "utilities.ts",
+        "scriptVersionCache.ts",
+        "scriptInfo.ts",
+        "lshost.ts",
+        "project.ts",
         "editorServices.ts",
         "protocol.d.ts",
         "session.ts"

src/server/typingsCache.ts

@@ -0,0 +1,94 @@
+/// <reference path="project.ts"/>
+
+namespace ts.server {
+    export interface ITypingsInstaller {
+        enqueueInstallTypingsRequest(p: Project): void;
+    }
+
+    class TypingsCacheEntry {
+        readonly typingOptions: TypingOptions;
+        readonly compilerOptions: CompilerOptions;
+        readonly typings: Path[];
+    }
+
+    const emptyArray: any[] = [];
+    const jsOrDts = [".js", ".d.ts"];
+
+    function getTypingOptionsForProjects(proj: Project): TypingOptions {
+        if (proj.projectKind === ProjectKind.Configured) {
+            return (<ConfiguredProject>proj).getTypingOptions();
+        }
+
+        const enableAutoDiscovery = 
+            proj.projectKind === ProjectKind.Inferred && 
+            proj.getCompilerOptions().allowJs &&
+            proj.getFileNames().every(f => fileExtensionIsAny(f, jsOrDts));
+
+        // TODO: add .d.ts files to excludes 
+        return { enableAutoDiscovery, include: emptyArray, exclude: emptyArray };
+    }
+
+    function setIsEqualTo(arr1: string[], arr2: string[]): boolean {
+        if (arr1 === arr2) {
+            return true;
+        }
+        if ((arr1 || emptyArray).length === 0 && (arr2 || emptyArray).length === 0) {
+            return true;
+        }
+        const set: Map<boolean> = Object.create(null);
+        let unique = 0;
+
+        for (const v of arr1) {
+            if (set[v] !== true) {
+                set[v] = true;
+                unique++;
+            }
+        }
+        for (const v of arr2) {
+            if (!hasProperty(set, v)) {
+                return false;
+            }
+            if (set[v] === true) {
+                set[v] = false;
+                unique--;
+            }
+        }
+        return unique === 0;
+    }
+
+    function typingOptionsChanged(opt1: TypingOptions, opt2: TypingOptions): boolean {
+        return opt1.enableAutoDiscovery !== opt2.enableAutoDiscovery ||
+            !setIsEqualTo(opt1.include, opt2.include) ||
+            !setIsEqualTo(opt1.exclude, opt2.exclude); 
+    }
+
+    function compilerOptionsChanged(opt1: CompilerOptions, opt2: CompilerOptions): boolean {
+        // TODO: add more relevant properties
+        return opt1.allowJs != opt2.allowJs;
+    }
+
+    export class TypingsCache {
+        private readonly perProjectCache: Map<TypingsCacheEntry> = {};
+
+        constructor(private readonly installer: ITypingsInstaller) {
+        }
+
+        getTypingsForProject(project: Project): Path[] {
+            const typingOptions = getTypingOptionsForProjects(project);
+
+            if(!typingOptions.enableAutoDiscovery) {
+                return emptyArray;
+            }
+
+            const entry = this.perProjectCache[project.getProjectName()];
+            if (!entry || typingOptionsChanged(typingOptions, entry.typingOptions) || compilerOptionsChanged(project.getCompilerOptions(), entry.compilerOptions)) {
+                this.installer.enqueueInstallTypingsRequest(project);
+            }
+            return entry? entry.typings : emptyArray;
+        }
+
+        deleteTypingsForProject(project: Project) {
+            delete this.perProjectCache[project.getProjectName()];
+        }
+    }
+}

src/server/typingsInstaller/nodeTypingsInstaller.ts

@@ -0,0 +1,49 @@
+/// <reference path="typingsInstaller.ts"/>
+
+namespace ts.server.typingsInstaller {
+    export class NodeTypingsInstaller extends TypingsInstaller {
+        private execSync: { (command: string, options: { stdio: "ignore" }): any };
+        constructor() {
+            super();
+            this.execSync = require("child_process").execSync;
+        }
+
+        init() {
+            super.init();
+            process.on("install", (req: InstallTypingsRequest) => {
+                this.install(req);
+            })
+        }
+
+        protected isPackageInstalled(packageName: string) {
+            try {
+                this.execSync(`npm list --global --depth=1 ${name}`, { stdio: "ignore" });
+                return true;
+            }
+            catch (e) {
+                return false;
+            }
+        }
+
+        protected installPackage(packageName: string) {
+            try {
+                this.execSync(`npm install --global ${name}`, { stdio: "ignore" });
+                return true;
+            }
+            catch (e) {
+                return false;
+            }
+        }
+
+        protected getTypingResolutionHost() {
+            return sys;
+        }
+
+        protected sendResponse(response: InstallTypingsResponse) {
+            process.send(response);
+        }
+    }
+
+    const installer = new NodeTypingsInstaller();
+    installer.init();
+}

src/server/typingsInstaller/tsconfig.json

@@ -0,0 +1,21 @@
+{
+    "compilerOptions": {
+        "noImplicitAny": true,
+        "noImplicitThis": true,
+        "removeComments": true,
+        "preserveConstEnums": true,
+        "pretty": true,
+        "out": "../../../built/local/typingsInstaller.js",
+        "sourceMap": true,
+        "stripInternal": true,
+        "types": [
+            "node"
+        ]
+    },
+    "files": [
+        "../../services/services.ts",
+        "../utilities.ts",
+        "typingsInstaller.ts",
+        "nodeTypingsInstaller.ts"
+    ]
+}

src/server/typingsInstaller/typingsInstaller.ts

@@ -0,0 +1,75 @@
+/// <reference path="../../services/services.ts"/>
+/// <reference path="../utilities.ts"/>
+
+
+
+namespace ts.server.typingsInstaller {
+    
+    const DefaultTsdSettings = JSON.stringify({
+        version: "v4",
+        repo: "DefinitelyTyped/DefinitelyTyped",
+        ref: "master",
+        path: "typings"
+    }, /*replacer*/undefined, /*space*/4);
+
+    export abstract class TypingsInstaller {
+        
+        private isTsdInstalled: boolean;
+        private missingTypings: Map<string> = {};
+
+        init() {
+            this.isTsdInstalled = this.isPackageInstalled("tsd");
+            if (!this.isTsdInstalled) {
+                this.isTsdInstalled = this.installPackage("tsd");
+            }
+        }
+
+        install(req: InstallTypingsRequest) {
+            if (!this.isTsdInstalled) {
+                return;
+            }
+
+            const discoverTypingsResult = JsTyping.discoverTypings(
+                this.getInstallTypingHost(),
+                req.fileNames,
+                req.projectRootPath,
+                req.safeListPath,
+                req.packageNameToTypingLocation,
+                req.typingOptions,
+                req.compilerOptions);
+
+            // respond with whatever cached typings we have now
+            this.sendResponse(this.createResponse(req, discoverTypingsResult.cachedTypingPaths));
+            this.watchFiles(discoverTypingsResult.filesToWatch);
+            this.installTypings(req, discoverTypingsResult.newTypingNames);
+        }
+
+        private installTypings(req: InstallTypingsRequest, typingsToInstall: string[]) {
+            // TODO: install typings and send response when they are ready
+            const existingTypings = typingsToInstall.fi
+            const host = this.getInstallTypingHost();
+            const tsdPath = combinePaths(req.cachePath, "tsd.json");
+            if (!host.fileExists(tsdPath)) {
+                host.writeFile(tsdPath, DefaultTsdSettings);
+            }
+        }
+
+        private watchFiles(files: string[]) {
+            // TODO: start watching files
+        }
+
+        private createResponse(request: InstallTypingsRequest, typings: string[]) {
+            return {
+                projectName: request.projectName,
+                typingOptions: request.typingOptions,
+                compilerOptions: request.compilerOptions,
+                typings
+            };
+        }
+
+        protected abstract isPackageInstalled(packageName: string): boolean;
+        protected abstract installPackage(packageName: string): boolean;
+        protected abstract getInstallTypingHost(): InstallTypingHost;
+        protected abstract sendResponse(response: InstallTypingsResponse): void;
+    }
+}

src/server/utilities.ts

@@ -8,6 +8,28 @@ namespace ts.server {
         verbose
     }
 
+    export interface InstallTypingsRequest {
+        readonly projectName: string;
+        readonly fileNames: string[];
+        readonly projectRootPath: ts.Path;
+        readonly safeListPath: ts.Path;
+        readonly packageNameToTypingLocation: ts.Map<string>;
+        readonly typingOptions: ts.TypingOptions;
+        readonly compilerOptions: ts.CompilerOptions;
+        readonly cachePath: string;
+    }
+
+    export interface InstallTypingsResponse {
+        readonly projectName: string;
+        readonly typingOptions: ts.TypingOptions;
+        readonly compilerOptions: ts.CompilerOptions;
+        readonly typings: string[];
+    }
+
+    export interface InstallTypingHost extends JsTyping.TypingResolutionHost {
+        writeFile(path: string, content: string): void;
+    }
+
     export interface Logger {
         close(): void;
         hasLevel(level: LogLevel): boolean;
@@ -188,6 +210,7 @@ namespace ts.server {
         files?: string[];
         wildcardDirectories?: Map<WatchDirectoryFlags>;
         compilerOptions?: CompilerOptions;
+        typingOptions?: TypingOptions;
     }
 
     export function isInferredProjectName(name: string) {
@@ -218,4 +241,35 @@ namespace ts.server {
             cb();
         }
     }
+
+    export class GcTimer {
+        private timerId: any;
+        private gc: () => void;
+        constructor(private readonly host: ServerHost, private readonly delay: number, private readonly logger: Logger) {
+            if (typeof global !== "undefined" && global.gc) {
+                this.gc = () => global.gc();
+            }
+        }
+
+        public scheduleCollect() {
+            if (!this.gc || this.timerId != undefined) {
+                // no global.gc or collection was already scheduled - skip this request
+                return;
+            }
+            this.timerId = this.host.setTimeout(GcTimer.run, this.delay, this);
+        }
+
+        private static run(self: GcTimer) {
+            self.timerId = undefined;
+
+            const log = self.logger.hasLevel(LogLevel.requestTime);
+            const before = log && self.host.getMemoryUsage();
+
+            self.gc();
+            if (log) {
+                const after = self.host.getMemoryUsage();
+                self.logger.perftrc(`GC::before ${before}, after ${after}`);
+            }
+        }
+    }
 }