diff --git a/Jakefile.js b/Jakefile.js index acc614421e6..474303a9722 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -680,9 +680,9 @@ function cleanTestDirs() { } // used to pass data from jake command line directly to run.js -function writeTestConfigFile(tests, light, testConfigFile) { - console.log('Running test(s): ' + tests); - var testConfigContents = JSON.stringify({ test: [tests], light: light }); +function writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, testConfigFile) { + var testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, light: light, workerCount: workerCount, taskConfigsFolder: taskConfigsFolder }); + console.log('Running tests with config: ' + testConfigContents); fs.writeFileSync('test.config', testConfigContents); } @@ -692,7 +692,7 @@ function deleteTemporaryProjectOutput() { } } -function runConsoleTests(defaultReporter, defaultSubsets) { +function runConsoleTests(defaultReporter, runInParallel) { cleanTestDirs(); var debug = process.env.debug || process.env.d; tests = process.env.test || process.env.tests || process.env.t; @@ -701,9 +701,22 @@ function runConsoleTests(defaultReporter, defaultSubsets) { if(fs.existsSync(testConfigFile)) { fs.unlinkSync(testConfigFile); } + var workerCount, taskConfigsFolder; + if (runInParallel) { + // generate name to store task configuration files + var prefix = os.tmpdir() + "/ts-tests"; + var i = 1; + do { + taskConfigsFolder = prefix + i; + i++; + } while (fs.existsSync(taskConfigsFolder)); + fs.mkdirSync(taskConfigsFolder); - if (tests || light) { - writeTestConfigFile(tests, light, testConfigFile); + workerCount = process.env.workerCount || os.cpus().length; + } + + if (tests || light || taskConfigsFolder) { + writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, testConfigFile); } if (tests && tests.toLocaleLowerCase() === "rwc") { @@ -717,61 +730,93 @@ function runConsoleTests(defaultReporter, defaultSubsets) { // timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally // default timeout is 2sec which really should be enough, but maybe we just need a small amount longer - var subsetRegexes; - if(defaultSubsets.length === 0) { - subsetRegexes = [tests]; - } - else { - var subsets = tests ? tests.split("|") : defaultSubsets; - subsetRegexes = subsets.map(function (sub) { return "^" + sub + ".*$"; }); - subsetRegexes.push("^(?!" + subsets.join("|") + ").*$"); - } - var counter = subsetRegexes.length; - var errorStatus; - subsetRegexes.forEach(function (subsetRegex, i) { - tests = subsetRegex ? ' -g "' + subsetRegex + '"' : ''; + if(!runInParallel) { + tests = tests ? ' -g "' + tests + '"' : ''; var cmd = "mocha" + (debug ? " --debug-brk" : "") + " -R " + reporter + tests + colors + ' -t ' + testTimeout + ' ' + run; console.log(cmd); - function finish(status) { - counter--; - // save first error status - if (status !== undefined && errorStatus === undefined) { - errorStatus = status; - } - - deleteTemporaryProjectOutput(); - if (counter !== 0 || errorStatus === undefined) { - // run linter when last worker is finished - if (lintFlag && counter === 0) { - var lint = jake.Task['lint']; - lint.addListener('complete', function () { - complete(); - }); - lint.invoke(); - } - complete(); - } - else { - fail("Process exited with code " + status); - } - } exec(cmd, function () { + runLinter(); finish(); }, function(e, status) { finish(status); }); - }); + + } + else { + // run task to load all tests and partition them between workers + var cmd = "mocha " + " -R min " + colors + run; + console.log(cmd); + exec(cmd, function() { + // read all configuration files and spawn a worker for every config + var configFiles = fs.readdirSync(taskConfigsFolder); + var counter = configFiles.length; + var firstErrorStatus; + // schedule work for chunks + configFiles.forEach(function (f) { + var configPath = path.join(taskConfigsFolder, f); + var workerCmd = "mocha" + " -t " + testTimeout + " -R " + reporter + " " + colors + " " + run + " --config='" + configPath + "'"; + console.log(workerCmd); + exec(workerCmd, finishWorker, finishWorker) + }); + + function finishWorker(e, errorStatus) { + counter--; + if (firstErrorStatus === undefined && errorStatus !== undefined) { + firstErrorStatus = errorStatus; + } + if (counter !== 0) { + complete(); + } + else { + // last worker clean everything and runs linter in case if there were no errors + deleteTemporaryProjectOutput(); + jake.rmRf(taskConfigsFolder); + if (firstErrorStatus === undefined) { + runLinter(); + complete(); + } + else { + failWithStatus(firstErrorStatus); + } + } + } + }); + } + + function failWithStatus(status) { + fail("Process exited with code " + status); + } + + function finish(errorStatus) { + deleteTemporaryProjectOutput(); + if (errorStatus !== undefined) { + failWithStatus(errorStatus); + } + else { + complete(); + } + } + function runLinter() { + if (!lintFlag) { + return; + } + var lint = jake.Task['lint']; + lint.addListener('complete', function () { + complete(); + }); + lint.invoke(); + } } var testTimeout = 20000; desc("Runs all the tests in parallel using the built run.js file. Optional arguments are: t[ests]=category1|category2|... d[ebug]=true."); task("runtests-parallel", ["build-rules", "tests", builtLocalDirectory], function() { - runConsoleTests('min', ['compiler', 'conformance', 'Projects', 'fourslash']); + runConsoleTests('min', /*runInParallel*/ true); }, {async: true}); desc("Runs the tests using the built run.js file. Optional arguments are: t[ests]=regex r[eporter]=[list|spec|json|] d[ebug]=true color[s]=false lint=true."); task("runtests", ["build-rules", "tests", builtLocalDirectory], function() { - runConsoleTests('mocha-fivemat-progress-reporter', []); + runConsoleTests('mocha-fivemat-progress-reporter', /*runInParallel*/ false); }, {async: true}); desc("Generates code coverage data via instanbul"); diff --git a/src/harness/compilerRunner.ts b/src/harness/compilerRunner.ts index 1a83f8c4582..e822b31584e 100644 --- a/src/harness/compilerRunner.ts +++ b/src/harness/compilerRunner.ts @@ -12,7 +12,7 @@ const enum CompilerTestType { class CompilerBaselineRunner extends RunnerBase { private basePath = "tests/cases"; - private testSuiteName: string; + private testSuiteName: TestRunnerKind; private errors: boolean; private emit: boolean; private decl: boolean; @@ -41,6 +41,14 @@ class CompilerBaselineRunner extends RunnerBase { this.basePath += "/" + this.testSuiteName; } + public kind() { + return this.testSuiteName; + } + + public enumerateTestFiles() { + return this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true }); + } + private makeUnitName(name: string, root: string) { return ts.isRootedDiskPath(name) ? name : ts.combinePaths(root, name); }; @@ -391,7 +399,7 @@ class CompilerBaselineRunner extends RunnerBase { // this will set up a series of describe/it blocks to run between the setup and cleanup phases if (this.tests.length === 0) { - const testFiles = this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true }); + const testFiles = this.enumerateTestFiles(); testFiles.forEach(fn => { fn = fn.replace(/\\/g, "/"); this.checkTestCodeOutput(fn); diff --git a/src/harness/fourslashRunner.ts b/src/harness/fourslashRunner.ts index 0047e851fc1..d835054a868 100644 --- a/src/harness/fourslashRunner.ts +++ b/src/harness/fourslashRunner.ts @@ -11,7 +11,7 @@ const enum FourSlashTestType { class FourSlashRunner extends RunnerBase { protected basePath: string; - protected testSuiteName: string; + protected testSuiteName: TestRunnerKind; constructor(private testType: FourSlashTestType) { super(); @@ -35,9 +35,17 @@ class FourSlashRunner extends RunnerBase { } } + public enumerateTestFiles() { + return this.enumerateFiles(this.basePath, /\.ts/i, { recursive: false }); + } + + public kind() { + return this.testSuiteName; + } + public initializeTests() { if (this.tests.length === 0) { - this.tests = this.enumerateFiles(this.basePath, /\.ts/i, { recursive: false }); + this.tests = this.enumerateTestFiles(); } describe(this.testSuiteName + " tests", () => { diff --git a/src/harness/projectsRunner.ts b/src/harness/projectsRunner.ts index d4e91d43efe..60a0813b05d 100644 --- a/src/harness/projectsRunner.ts +++ b/src/harness/projectsRunner.ts @@ -36,11 +36,19 @@ interface BatchCompileProjectTestCaseResult extends CompileProjectFilesResult { } class ProjectRunner extends RunnerBase { + + public enumerateTestFiles() { + return this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true }); + } + + public kind(): TestRunnerKind { + return "project"; + } + public initializeTests() { if (this.tests.length === 0) { - const testFiles = this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true }); + const testFiles = this.enumerateTestFiles(); testFiles.forEach(fn => { - fn = fn.replace(/\\/g, "/"); this.runProjectTestCase(fn); }); } diff --git a/src/harness/runner.ts b/src/harness/runner.ts index b56959d7e5d..1c36614e61f 100644 --- a/src/harness/runner.ts +++ b/src/harness/runner.ts @@ -31,18 +31,90 @@ function runTests(runners: RunnerBase[]) { } } -// users can define tests to run in mytest.config that will override cmd line args, otherwise use cmd line args (test.config), otherwise no options -let mytestconfig = "mytest.config"; -let testconfig = "test.config"; -let testConfigFile = - Harness.IO.fileExists(mytestconfig) ? Harness.IO.readFile(mytestconfig) : - (Harness.IO.fileExists(testconfig) ? Harness.IO.readFile(testconfig) : ""); +function tryGetConfig(args: string[]) { + const prefix = "--config="; + const configPath = ts.forEach(args, arg => arg.lastIndexOf(prefix, 0) === 0 && arg.substr(prefix.length)); + // strip leading and trailing quotes from the path (necessary on Windows since shell does not do it automatically) + return configPath && configPath.replace(/(^[\"'])|([\"']$)/g, ""); +} -if (testConfigFile !== "") { - const testConfig = JSON.parse(testConfigFile); +function createRunner(kind: TestRunnerKind): RunnerBase { + switch (kind) { + case "conformance": + return new CompilerBaselineRunner(CompilerTestType.Conformance); + case "compiler": + return new CompilerBaselineRunner(CompilerTestType.Regressions); + case "fourslash": + return new FourSlashRunner(FourSlashTestType.Native); + case "fourslash-shims": + return new FourSlashRunner(FourSlashTestType.Shims); + case "fourslash-shims-pp": + return new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess); + case "fourslash-server": + return new FourSlashRunner(FourSlashTestType.Server); + case "project": + return new ProjectRunner(); + case "rwc": + return new RWCRunner(); + case "test262": + return new Test262BaselineRunner(); + } +} + +// users can define tests to run in mytest.config that will override cmd line args, otherwise use cmd line args (test.config), otherwise no options + +const mytestconfigFileName = "mytest.config"; +const testconfigFileName = "test.config"; + +const customConfig = tryGetConfig(Harness.IO.args()); +let testConfigContent = + customConfig && Harness.IO.fileExists(customConfig) + ? Harness.IO.readFile(customConfig) + : Harness.IO.fileExists(mytestconfigFileName) + ? Harness.IO.readFile(mytestconfigFileName) + : Harness.IO.fileExists(testconfigFileName) ? Harness.IO.readFile(testconfigFileName) : ""; + +let taskConfigsFolder: string; +let workerCount: number; +let runUnitTests = true; + +interface TestConfig { + light?: boolean; + taskConfigsFolder?: string; + workerCount?: number; + tasks?: TaskSet[]; + test?: string[]; + runUnitTests?: boolean; +} + +interface TaskSet { + runner: TestRunnerKind; + files: string[]; +} + +if (testConfigContent !== "") { + const testConfig = JSON.parse(testConfigContent); if (testConfig.light) { Harness.lightMode = true; } + if (testConfig.taskConfigsFolder) { + taskConfigsFolder = testConfig.taskConfigsFolder; + } + if (testConfig.runUnitTests !== undefined) { + runUnitTests = testConfig.runUnitTests; + } + if (testConfig.workerCount) { + workerCount = testConfig.workerCount; + } + if (testConfig.tasks) { + for (const taskSet of testConfig.tasks) { + const runner = createRunner(taskSet.runner); + for (const file of taskSet.files) { + runner.addTest(file); + } + runners.push(runner); + } + } if (testConfig.test && testConfig.test.length > 0) { for (const option of testConfig.test) { @@ -106,4 +178,41 @@ if (runners.length === 0) { // runners.push(new GeneratedFourslashRunner()); } -runTests(runners); +if (taskConfigsFolder) { + // this instance of mocha should only partition work but not run actual tests + runUnitTests = false; + const workerConfigs: TestConfig[] = []; + for (let i = 0; i < workerCount; i++) { + // pass light mode settings to workers + workerConfigs.push({ light: Harness.lightMode, tasks: [] }); + } + + for (const runner of runners) { + const files = runner.enumerateTestFiles(); + const chunkSize = Math.floor(files.length / workerCount) + 1; // add extra 1 to prevent missing tests due to rounding + for (let i = 0; i < workerCount; i++) { + const startPos = i * chunkSize; + const len = Math.min(chunkSize, files.length - startPos); + if (len !== 0) { + workerConfigs[i].tasks.push({ + runner: runner.kind(), + files: files.slice(startPos, startPos + len) + }); + } + } + } + + for (let i = 0; i < workerCount; i++) { + const config = workerConfigs[i]; + // use last worker to run unit tests + config.runUnitTests = i === workerCount - 1; + Harness.IO.writeFile(ts.combinePaths(taskConfigsFolder, `task-config${i}.json`), JSON.stringify(workerConfigs[i])); + } +} +else { + runTests(runners); +} +if (!runUnitTests) { + // patch `describe` to skip unit tests + describe = describe.skip; +} \ No newline at end of file diff --git a/src/harness/runnerbase.ts b/src/harness/runnerbase.ts index a114e8c3201..81da0907714 100644 --- a/src/harness/runnerbase.ts +++ b/src/harness/runnerbase.ts @@ -1,5 +1,10 @@ /// + +type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262"; +type CompilerTestKind = "conformance" | "compiler"; +type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server"; + abstract class RunnerBase { constructor() { } @@ -12,9 +17,13 @@ abstract class RunnerBase { } public enumerateFiles(folder: string, regex?: RegExp, options?: { recursive: boolean }): string[] { - return Harness.IO.listFiles(Harness.userSpecifiedRoot + folder, regex, { recursive: (options ? options.recursive : false) }); + return ts.map(Harness.IO.listFiles(Harness.userSpecifiedRoot + folder, regex, { recursive: (options ? options.recursive : false) }), ts.normalizeSlashes); } + abstract kind(): TestRunnerKind; + + abstract enumerateTestFiles(): string[]; + /** Setup the runner's tests so that they are ready to be executed by the harness * The first test should be a describe/it block that sets up the harness's compiler instance appropriately */ diff --git a/src/harness/rwcRunner.ts b/src/harness/rwcRunner.ts index 99b1ce8cc88..2ca69b09126 100644 --- a/src/harness/rwcRunner.ts +++ b/src/harness/rwcRunner.ts @@ -224,12 +224,20 @@ namespace RWC { class RWCRunner extends RunnerBase { private static sourcePath = "internal/cases/rwc/"; + public enumerateTestFiles() { + return Harness.IO.listFiles(RWCRunner.sourcePath, /.+\.json$/); + } + + public kind(): TestRunnerKind { + return "rwc"; + } + /** Setup the runner's tests so that they are ready to be executed by the harness * The first test should be a describe/it block that sets up the harness's compiler instance appropriately */ public initializeTests(): void { // Read in and evaluate the test list - const testList = Harness.IO.listFiles(RWCRunner.sourcePath, /.+\.json$/); + const testList = this.enumerateTestFiles(); for (let i = 0; i < testList.length; i++) { this.runTest(testList[i]); } diff --git a/src/harness/test262Runner.ts b/src/harness/test262Runner.ts index 241af5f3d00..c44b2286b83 100644 --- a/src/harness/test262Runner.ts +++ b/src/harness/test262Runner.ts @@ -98,12 +98,20 @@ class Test262BaselineRunner extends RunnerBase { }); } + public kind(): TestRunnerKind { + return "test262"; + } + + public enumerateTestFiles() { + return ts.map(this.enumerateFiles(Test262BaselineRunner.basePath, Test262BaselineRunner.testFileExtensionRegex, { recursive: true }), ts.normalizePath); + } + public initializeTests() { // this will set up a series of describe/it blocks to run between the setup and cleanup phases if (this.tests.length === 0) { - const testFiles = this.enumerateFiles(Test262BaselineRunner.basePath, Test262BaselineRunner.testFileExtensionRegex, { recursive: true }); + const testFiles = this.enumerateTestFiles(); testFiles.forEach(fn => { - this.runTest(ts.normalizePath(fn)); + this.runTest(fn); }); } else {