Add DefinitelyTyped test runner

Assumes that ../DefinitelyTyped holds the DefinitelyTyped repo.

src/harness/definitelyRunner.ts

@@ -0,0 +1,51 @@
+/// <reference path="harness.ts"/>
+/// <reference path="runnerbase.ts" />
+class DefinitelyTypedRunner extends RunnerBase {
+    private static readonly testDir = "../DefinitelyTyped/types/";
+    public enumerateTestFiles() {
+        return Harness.IO.getDirectories(DefinitelyTypedRunner.testDir).map(dir => DefinitelyTypedRunner.testDir + dir);
+    }
+
+    public kind(): TestRunnerKind {
+        return "definitely";
+    }
+
+    /** Setup the runner's tests so that they are ready to be executed by the harness
+     *  The first test should be a describe/it block that sets up the harness's compiler instance appropriately
+     */
+    public initializeTests(): void {
+        // Read in and evaluate the test list
+        const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles();
+
+        describe(`${this.kind()} code samples`, () => {
+            for (const test of testList) {
+                this.runTest(test);
+            }
+        });
+    }
+
+    private runTest(directoryName: string) {
+        describe(directoryName, () => {
+            const cp = require("child_process");
+            const path = require("path");
+
+            it("should build successfully", () => {
+                const cwd = path.join(__dirname, "../../", directoryName);
+                const timeout = 600000; // 600s = 10 minutes
+                const stdio = isWorker ? "pipe" : "inherit";
+                const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio });
+                if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`);
+                Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => {
+                    const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js"), "--lib dom,es6", "--strict"], { cwd, timeout, shell: true });
+                    return `Exit Code: ${result.status}
+Standard output:
+${result.stdout.toString().replace(/\r\n/g, "\n")}
+
+
+Standard error:
+${result.stderr.toString().replace(/\r\n/g, "\n")}`;
+                });
+            });
+        });
+    }
+}

src/harness/runner.ts

@@ -19,6 +19,7 @@
 /// <reference path="projectsRunner.ts" />
 /// <reference path="rwcRunner.ts" />
 /// <reference path="userRunner.ts" />
+/// <reference path="definitelyRunner.ts" />
 /// <reference path="harness.ts" />
 /// <reference path="./parallel/shared.ts" />
 
@@ -62,6 +63,8 @@ function createRunner(kind: TestRunnerKind): RunnerBase {
             return new Test262BaselineRunner();
         case "user":
             return new UserCodeRunner();
+        case "definitely":
+            return new DefinitelyTypedRunner();
     }
     ts.Debug.fail(`Unknown runner kind ${kind}`);
 }
@@ -183,6 +186,9 @@ function handleTestConfig() {
                     case "user":
                         runners.push(new UserCodeRunner());
                         break;
+                    case "definitely":
+                        runners.push(new DefinitelyTypedRunner());
+                        break;
                 }
             }
         }

src/harness/runnerbase.ts

@@ -1,7 +1,7 @@
 /// <reference path="harness.ts" />
 
 
-type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user";
+type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user" | "definitely";
 type CompilerTestKind = "conformance" | "compiler";
 type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server";
 

src/harness/tsconfig.json

@@ -93,6 +93,7 @@
         "loggedIO.ts",
         "rwcRunner.ts",
         "userRunner.ts",
+        "definitelyRunner.ts",
         "test262Runner.ts",
         "./parallel/shared.ts",
         "./parallel/host.ts",

src/harness/userRunner.ts

@@ -36,7 +36,7 @@ class UserCodeRunner extends RunnerBase {
                 const install = cp.spawnSync(`npm`, ["i"], { cwd, timeout, shell: true, stdio });
                 if (install.status !== 0) throw new Error(`NPM Install for ${directoryName} failed!`);
                 Harness.Baseline.runBaseline(`${this.kind()}/${directoryName}.log`, () => {
-                    const result = cp.spawnSync(`node`, ["../../../../built/local/tsc.js"], { cwd, timeout, shell: true });
+                    const result = cp.spawnSync(`node`, [path.join(__dirname, "tsc.js")], { cwd, timeout, shell: true });
                     return `Exit Code: ${result.status}
 Standard output:
 ${result.stdout.toString().replace(/\r\n/g, "\n")}