From f69ecb5b90746ad6e87c6cff8044eb02f7e1b7ca Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Mon, 23 May 2016 16:00:56 -0700 Subject: [PATCH 1/3] run tests in parallel by equally dividing them between workers --- Jakefile.js | 99 +++++++++++++++++-------- src/harness/compilerRunner.ts | 12 +++- src/harness/fourslashRunner.ts | 12 +++- src/harness/projectsRunner.ts | 12 +++- src/harness/runner.ts | 127 ++++++++++++++++++++++++++++++--- src/harness/runnerbase.ts | 11 ++- src/harness/rwcRunner.ts | 10 ++- src/harness/test262Runner.ts | 12 +++- 8 files changed, 246 insertions(+), 49 deletions(-) diff --git a/Jakefile.js b/Jakefile.js index 5112a8db597..73bfdfdaedf 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -680,9 +680,9 @@ function cleanTestDirs() { } // used to pass data from jake command line directly to run.js -function writeTestConfigFile(tests, light, testConfigFile) { - console.log('Running test(s): ' + tests); - var testConfigContents = JSON.stringify({ test: [tests], light: light }); +function writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, testConfigFile) { + var testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, light: light, workerCount: workerCount, taskConfigsFolder: taskConfigsFolder }); + console.log('Running tests with config: ' + testConfigContents); fs.writeFileSync('test.config', testConfigContents); } @@ -692,7 +692,7 @@ function deleteTemporaryProjectOutput() { } } -function runConsoleTests(defaultReporter, defaultSubsets) { +function runConsoleTests(defaultReporter, runInParallel) { cleanTestDirs(); var debug = process.env.debug || process.env.d; tests = process.env.test || process.env.tests || process.env.t; @@ -701,9 +701,22 @@ function runConsoleTests(defaultReporter, defaultSubsets) { if(fs.existsSync(testConfigFile)) { fs.unlinkSync(testConfigFile); } + var workerCount, taskConfigsFolder; + if (runInParallel) { + // generate name to store task configuration files + var prefix = os.tmpdir() + "/ts-tests"; + var i = 1; + do { + taskConfigsFolder = prefix + i; + i++; + } while (fs.existsSync(taskConfigsFolder)); + fs.mkdirSync(taskConfigsFolder); - if (tests || light) { - writeTestConfigFile(tests, light, testConfigFile); + workerCount = process.env.workerCount || os.cpus().length; + } + + if (tests || light || taskConfigsFolder) { + writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, testConfigFile); } if (tests && tests.toLocaleLowerCase() === "rwc") { @@ -717,45 +730,71 @@ function runConsoleTests(defaultReporter, defaultSubsets) { // timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally // default timeout is 2sec which really should be enough, but maybe we just need a small amount longer - var subsetRegexes; - if(defaultSubsets.length === 0) { - subsetRegexes = [tests]; - } - else { - var subsets = tests ? tests.split("|") : defaultSubsets; - subsetRegexes = subsets.map(function (sub) { return "^" + sub + ".*$"; }); - subsetRegexes.push("^(?!" + subsets.join("|") + ").*$"); - } - subsetRegexes.forEach(function (subsetRegex, i) { - tests = subsetRegex ? ' -g "' + subsetRegex + '"' : ''; + if(!runInParallel) { + tests = tests ? ' -g "' + tests + '"' : ''; var cmd = "mocha" + (debug ? " --debug-brk" : "") + " -R " + reporter + tests + colors + ' -t ' + testTimeout + ' ' + run; console.log(cmd); - function finish() { - deleteTemporaryProjectOutput(); - complete(); - } exec(cmd, function () { - if (lintFlag && i === 0) { - var lint = jake.Task['lint']; - lint.addListener('complete', function () { - complete(); - }); - lint.invoke(); + if (i === 0) { + runLinter(); } finish(); }, finish); - }); + + } + else { + // run task to load all tests and partition then between workers + var cmd = "mocha " + " -R min " + colors + run; + console.log(cmd); + exec(cmd, function() { + // read all configuration files and spawn a worker for every config + var configFiles = fs.readdirSync(taskConfigsFolder); + var counter = configFiles.length; + // schedule work for chunks + configFiles.forEach(function (f) { + var configPath = path.join(taskConfigsFolder, f); + var workerCmd = "mocha" + " -t " + testTimeout + " -R " + reporter + " " + colors + " " + run + " --config='" + configPath + "'"; + console.log(workerCmd); + exec(workerCmd, finishWorker, finishWorker) + }); + + function finishWorker() { + counter--; + if (counter === 0) { + // last worker clean everything and runs linter + runLinter(); + deleteTemporaryProjectOutput(); + jake.rmRf(taskConfigsFolder); + } + complete(); + } + }); + } + function finish() { + deleteTemporaryProjectOutput(); + complete(); + } + function runLinter() { + if (!lintFlag) { + return; + } + var lint = jake.Task['lint']; + lint.addListener('complete', function () { + complete(); + }); + lint.invoke(); + } } var testTimeout = 20000; desc("Runs all the tests in parallel using the built run.js file. Optional arguments are: t[ests]=category1|category2|... d[ebug]=true."); task("runtests-parallel", ["build-rules", "tests", builtLocalDirectory], function() { - runConsoleTests('min', ['compiler', 'conformance', 'Projects', 'fourslash']); + runConsoleTests('min', /*runInParallel*/ true); }, {async: true}); desc("Runs the tests using the built run.js file. Optional arguments are: t[ests]=regex r[eporter]=[list|spec|json|] d[ebug]=true color[s]=false lint=true."); task("runtests", ["build-rules", "tests", builtLocalDirectory], function() { - runConsoleTests('mocha-fivemat-progress-reporter', []); + runConsoleTests('mocha-fivemat-progress-reporter', /*runInParallel*/ false); }, {async: true}); desc("Generates code coverage data via instanbul"); diff --git a/src/harness/compilerRunner.ts b/src/harness/compilerRunner.ts index 1ddbd1ea331..fefea00b807 100644 --- a/src/harness/compilerRunner.ts +++ b/src/harness/compilerRunner.ts @@ -11,7 +11,7 @@ const enum CompilerTestType { class CompilerBaselineRunner extends RunnerBase { private basePath = "tests/cases"; - private testSuiteName: string; + private testSuiteName: TestRunnerKind; private errors: boolean; private emit: boolean; private decl: boolean; @@ -40,6 +40,14 @@ class CompilerBaselineRunner extends RunnerBase { this.basePath += "/" + this.testSuiteName; } + public kind() { + return this.testSuiteName; + } + + public enumerateTestFiles() { + return this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true }); + } + private makeUnitName(name: string, root: string) { return ts.isRootedDiskPath(name) ? name : ts.combinePaths(root, name); }; @@ -390,7 +398,7 @@ class CompilerBaselineRunner extends RunnerBase { // this will set up a series of describe/it blocks to run between the setup and cleanup phases if (this.tests.length === 0) { - const testFiles = this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true }); + const testFiles = this.enumerateTestFiles(); testFiles.forEach(fn => { fn = fn.replace(/\\/g, "/"); this.checkTestCodeOutput(fn); diff --git a/src/harness/fourslashRunner.ts b/src/harness/fourslashRunner.ts index 386bd9e340c..a6174342acd 100644 --- a/src/harness/fourslashRunner.ts +++ b/src/harness/fourslashRunner.ts @@ -12,7 +12,7 @@ const enum FourSlashTestType { class FourSlashRunner extends RunnerBase { protected basePath: string; - protected testSuiteName: string; + protected testSuiteName: TestRunnerKind; constructor(private testType: FourSlashTestType) { super(); @@ -36,9 +36,17 @@ class FourSlashRunner extends RunnerBase { } } + public enumerateTestFiles() { + return this.enumerateFiles(this.basePath, /\.ts/i, { recursive: false }); + } + + public kind() { + return this.testSuiteName; + } + public initializeTests() { if (this.tests.length === 0) { - this.tests = this.enumerateFiles(this.basePath, /\.ts/i, { recursive: false }); + this.tests = this.enumerateTestFiles(); } describe(this.testSuiteName + " tests", () => { diff --git a/src/harness/projectsRunner.ts b/src/harness/projectsRunner.ts index 9ee4359d8d3..777ef63145d 100644 --- a/src/harness/projectsRunner.ts +++ b/src/harness/projectsRunner.ts @@ -37,11 +37,19 @@ interface BatchCompileProjectTestCaseResult extends CompileProjectFilesResult { } class ProjectRunner extends RunnerBase { + + public enumerateTestFiles() { + return this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true }); + } + + public kind(): TestRunnerKind { + return "project"; + } + public initializeTests() { if (this.tests.length === 0) { - const testFiles = this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true }); + const testFiles = this.enumerateTestFiles(); testFiles.forEach(fn => { - fn = fn.replace(/\\/g, "/"); this.runProjectTestCase(fn); }); } diff --git a/src/harness/runner.ts b/src/harness/runner.ts index bb3cafea0a8..07723913040 100644 --- a/src/harness/runner.ts +++ b/src/harness/runner.ts @@ -33,18 +33,90 @@ function runTests(runners: RunnerBase[]) { } } -// users can define tests to run in mytest.config that will override cmd line args, otherwise use cmd line args (test.config), otherwise no options -let mytestconfig = "mytest.config"; -let testconfig = "test.config"; -let testConfigFile = - Harness.IO.fileExists(mytestconfig) ? Harness.IO.readFile(mytestconfig) : - (Harness.IO.fileExists(testconfig) ? Harness.IO.readFile(testconfig) : ""); +function tryGetConfig(args: string[]) { + const prefix = "--config="; + const configPath = ts.forEach(args, arg => arg.lastIndexOf(prefix, 0) === 0 && arg.substr(prefix.length)); + // strip leading and trailing quotes from the path (necessary on Windows since shell does not do it automatically) + return configPath && configPath.replace(/(^[\"'])|([\"']$)/g, ""); +} -if (testConfigFile !== "") { - const testConfig = JSON.parse(testConfigFile); +function createRunner(kind: TestRunnerKind): RunnerBase { + switch (kind) { + case "conformance": + return new CompilerBaselineRunner(CompilerTestType.Conformance); + case "compiler": + return new CompilerBaselineRunner(CompilerTestType.Regressions); + case "fourslash": + return new FourSlashRunner(FourSlashTestType.Native); + case "fourslash-shims": + return new FourSlashRunner(FourSlashTestType.Shims); + case "fourslash-shims-pp": + return new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess); + case "fourslash-server": + return new FourSlashRunner(FourSlashTestType.Server); + case "project": + return new ProjectRunner(); + case "rwc": + return new RWCRunner(); + case "test262": + return new Test262BaselineRunner(); + } +} + +// users can define tests to run in mytest.config that will override cmd line args, otherwise use cmd line args (test.config), otherwise no options + +const mytestconfigFileName = "mytest.config"; +const testconfigFileName = "test.config"; + +const customConfig = tryGetConfig(Harness.IO.args()); +let testConfigContent = + customConfig && Harness.IO.fileExists(customConfig) + ? Harness.IO.readFile(customConfig) + : Harness.IO.fileExists(mytestconfigFileName) + ? Harness.IO.readFile(mytestconfigFileName) + : Harness.IO.fileExists(testconfigFileName) ? Harness.IO.readFile(testconfigFileName) : ""; + +let taskConfigsFolder: string; +let workerCount: number; +let runUnitTests = true; + +interface TestConfig { + light?: boolean; + taskConfigsFolder?: string; + workerCount?: number; + tasks?: TaskSet[]; + test?: string[]; + runUnitTests?: boolean; +} + +interface TaskSet { + runner: TestRunnerKind; + files: string[]; +} + +if (testConfigContent !== "") { + const testConfig = JSON.parse(testConfigContent); if (testConfig.light) { Harness.lightMode = true; } + if (testConfig.taskConfigsFolder) { + taskConfigsFolder = testConfig.taskConfigsFolder; + } + if (testConfig.runUnitTests !== undefined) { + runUnitTests = testConfig.runUnitTests; + } + if (testConfig.workerCount) { + workerCount = testConfig.workerCount; + } + if (testConfig.tasks) { + for (const taskSet of testConfig.tasks) { + const runner = createRunner(taskSet.runner); + for (const file of taskSet.files) { + runner.addTest(file); + } + runners.push(runner); + } + } if (testConfig.test && testConfig.test.length > 0) { for (const option of testConfig.test) { @@ -108,4 +180,41 @@ if (runners.length === 0) { // runners.push(new GeneratedFourslashRunner()); } -runTests(runners); +if (taskConfigsFolder) { + // this instance of mocha should only partition work but not run actual tests + runUnitTests = false; + const workerConfigs: TestConfig[] = []; + for (let i = 0; i < workerCount; i++) { + // pass light mode settings to workers + workerConfigs.push({ light: Harness.lightMode, tasks: [] }); + } + + for (const runner of runners) { + const files = runner.enumerateTestFiles(); + const chunkSize = Math.floor(files.length / workerCount) + 1; // add extra 1 to prevent missing tests due to rounding + for (let i = 0; i < workerCount; i++) { + const startPos = i * chunkSize; + const len = Math.min(chunkSize, files.length - startPos); + if (len !== 0) { + workerConfigs[i].tasks.push({ + runner: runner.kind(), + files: files.slice(startPos, startPos + len) + }); + } + } + } + + for (let i = 0; i < workerCount; i++) { + const config = workerConfigs[i]; + // use last worker to run unit tests + config.runUnitTests = i === workerCount - 1; + Harness.IO.writeFile(ts.combinePaths(taskConfigsFolder, `task-config${i}.json`), JSON.stringify(workerConfigs[i])); + } +} +else { + runTests(runners); +} +if (!runUnitTests) { + // patch `describe` to skip unit tests + describe = describe.skip; +} \ No newline at end of file diff --git a/src/harness/runnerbase.ts b/src/harness/runnerbase.ts index a114e8c3201..81da0907714 100644 --- a/src/harness/runnerbase.ts +++ b/src/harness/runnerbase.ts @@ -1,5 +1,10 @@ /// + +type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262"; +type CompilerTestKind = "conformance" | "compiler"; +type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server"; + abstract class RunnerBase { constructor() { } @@ -12,9 +17,13 @@ abstract class RunnerBase { } public enumerateFiles(folder: string, regex?: RegExp, options?: { recursive: boolean }): string[] { - return Harness.IO.listFiles(Harness.userSpecifiedRoot + folder, regex, { recursive: (options ? options.recursive : false) }); + return ts.map(Harness.IO.listFiles(Harness.userSpecifiedRoot + folder, regex, { recursive: (options ? options.recursive : false) }), ts.normalizeSlashes); } + abstract kind(): TestRunnerKind; + + abstract enumerateTestFiles(): string[]; + /** Setup the runner's tests so that they are ready to be executed by the harness * The first test should be a describe/it block that sets up the harness's compiler instance appropriately */ diff --git a/src/harness/rwcRunner.ts b/src/harness/rwcRunner.ts index 39231435bf5..ff1623b9051 100644 --- a/src/harness/rwcRunner.ts +++ b/src/harness/rwcRunner.ts @@ -223,12 +223,20 @@ namespace RWC { class RWCRunner extends RunnerBase { private static sourcePath = "internal/cases/rwc/"; + public enumerateTestFiles() { + return Harness.IO.listFiles(RWCRunner.sourcePath, /.+\.json$/); + } + + public kind(): TestRunnerKind { + return "rwc"; + } + /** Setup the runner's tests so that they are ready to be executed by the harness * The first test should be a describe/it block that sets up the harness's compiler instance appropriately */ public initializeTests(): void { // Read in and evaluate the test list - const testList = Harness.IO.listFiles(RWCRunner.sourcePath, /.+\.json$/); + const testList = this.enumerateTestFiles(); for (let i = 0; i < testList.length; i++) { this.runTest(testList[i]); } diff --git a/src/harness/test262Runner.ts b/src/harness/test262Runner.ts index cc9957c1fac..a3276cc65b3 100644 --- a/src/harness/test262Runner.ts +++ b/src/harness/test262Runner.ts @@ -97,12 +97,20 @@ class Test262BaselineRunner extends RunnerBase { }); } + public kind(): TestRunnerKind { + return "test262"; + } + + public enumerateTestFiles() { + return ts.map(this.enumerateFiles(Test262BaselineRunner.basePath, Test262BaselineRunner.testFileExtensionRegex, { recursive: true }), ts.normalizePath); + } + public initializeTests() { // this will set up a series of describe/it blocks to run between the setup and cleanup phases if (this.tests.length === 0) { - const testFiles = this.enumerateFiles(Test262BaselineRunner.basePath, Test262BaselineRunner.testFileExtensionRegex, { recursive: true }); + const testFiles = this.enumerateTestFiles(); testFiles.forEach(fn => { - this.runTest(ts.normalizePath(fn)); + this.runTest(fn); }); } else { From 5f7bbbf8fdb949c32e1d68e2dc1efb45a59f4a1e Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Tue, 24 May 2016 14:29:52 -0700 Subject: [PATCH 2/3] unconditionnaly use linter for non-parallel run --- Jakefile.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Jakefile.js b/Jakefile.js index 73bfdfdaedf..7be070bc492 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -735,9 +735,7 @@ function runConsoleTests(defaultReporter, runInParallel) { var cmd = "mocha" + (debug ? " --debug-brk" : "") + " -R " + reporter + tests + colors + ' -t ' + testTimeout + ' ' + run; console.log(cmd); exec(cmd, function () { - if (i === 0) { - runLinter(); - } + runLinter(); finish(); }, finish); From eab25119898cde36f1e7c1bc8e35a1aa52696b5c Mon Sep 17 00:00:00 2001 From: Vladimir Matveev Date: Fri, 27 May 2016 09:51:30 -0700 Subject: [PATCH 3/3] update signature of finishWorker --- Jakefile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jakefile.js b/Jakefile.js index f0415635956..474303a9722 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -759,7 +759,7 @@ function runConsoleTests(defaultReporter, runInParallel) { exec(workerCmd, finishWorker, finishWorker) }); - function finishWorker(errorStatus) { + function finishWorker(e, errorStatus) { counter--; if (firstErrorStatus === undefined && errorStatus !== undefined) { firstErrorStatus = errorStatus;