microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2026-05-10 21:07:52 -05:00
Code Issues Packages Projects Releases 212 Wiki Activity

Merge pull request #8774 from Microsoft/parallel-tests

Browse Source 
run tests in parallel by equally dividing them between workers
This commit is contained in:
Vladimir Matveev
2016-05-27 10:38:29 -07:00
parent 35156527a3 eab2511989
commit f07b4ba9d7
8 changed files with 267 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
Jakefile.js
View File

@@ -680,9 +680,9 @@ function cleanTestDirs() {
}
// used to pass data from jake command line directly to run.js
function writeTestConfigFile(tests, light, testConfigFile) {
console.log('Running test(s): ' + tests);
var testConfigContents = JSON.stringify({ test: [tests], light: light });
function writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, testConfigFile) {
var testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, light: light, workerCount: workerCount, taskConfigsFolder: taskConfigsFolder });
console.log('Running tests with config: ' + testConfigContents);
fs.writeFileSync('test.config', testConfigContents);
}
@@ -692,7 +692,7 @@ function deleteTemporaryProjectOutput() {
}
}
function runConsoleTests(defaultReporter, defaultSubsets) {
function runConsoleTests(defaultReporter, runInParallel) {
cleanTestDirs();
var debug = process.env.debug || process.env.d;
tests = process.env.test || process.env.tests || process.env.t;
@@ -701,9 +701,22 @@ function runConsoleTests(defaultReporter, defaultSubsets) {
if(fs.existsSync(testConfigFile)) {
fs.unlinkSync(testConfigFile);
}
var workerCount, taskConfigsFolder;
if (runInParallel) {
// generate name to store task configuration files
var prefix = os.tmpdir() + "/ts-tests";
var i = 1;
do {
taskConfigsFolder = prefix + i;
i++;
} while (fs.existsSync(taskConfigsFolder));
fs.mkdirSync(taskConfigsFolder);
if (tests || light) {
writeTestConfigFile(tests, light, testConfigFile);
workerCount = process.env.workerCount || os.cpus().length;
}
if (tests || light || taskConfigsFolder) {
writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, testConfigFile);
}
if (tests && tests.toLocaleLowerCase() === "rwc") {
@@ -717,61 +730,93 @@ function runConsoleTests(defaultReporter, defaultSubsets) {
// timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally
// default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
var subsetRegexes;
if(defaultSubsets.length === 0) {
subsetRegexes = [tests];
}
else {
var subsets = tests ? tests.split("|") : defaultSubsets;
subsetRegexes = subsets.map(function (sub) { return "^" + sub + ".*$"; });
subsetRegexes.push("^(?!" + subsets.join("|") + ").*$");
}
var counter = subsetRegexes.length;
var errorStatus;
subsetRegexes.forEach(function (subsetRegex, i) {
tests = subsetRegex ? ' -g "' + subsetRegex + '"' : '';
if(!runInParallel) {
tests = tests ? ' -g "' + tests + '"' : '';
var cmd = "mocha" + (debug ? " --debug-brk" : "") + " -R " + reporter + tests + colors + ' -t ' + testTimeout + ' ' + run;
console.log(cmd);
function finish(status) {
counter--;
// save first error status
if (status !== undefined && errorStatus === undefined) {
errorStatus = status;
}
deleteTemporaryProjectOutput();
if (counter !== 0 || errorStatus === undefined) {
// run linter when last worker is finished
if (lintFlag && counter === 0) {
var lint = jake.Task['lint'];
lint.addListener('complete', function () {
complete();
});
lint.invoke();
}
complete();
}
else {
fail("Process exited with code " + status);
}
}
exec(cmd, function () {
runLinter();
finish();
}, function(e, status) {
finish(status);
});
});
}
else {
// run task to load all tests and partition them between workers
var cmd = "mocha " + " -R min " + colors + run;
console.log(cmd);
exec(cmd, function() {
// read all configuration files and spawn a worker for every config
var configFiles = fs.readdirSync(taskConfigsFolder);
var counter = configFiles.length;
var firstErrorStatus;
// schedule work for chunks
configFiles.forEach(function (f) {
var configPath = path.join(taskConfigsFolder, f);
var workerCmd = "mocha" + " -t " + testTimeout + " -R " + reporter + " " + colors + " " + run + " --config='" + configPath + "'";
console.log(workerCmd);
exec(workerCmd, finishWorker, finishWorker)
});
function finishWorker(e, errorStatus) {
counter--;
if (firstErrorStatus === undefined && errorStatus !== undefined) {
firstErrorStatus = errorStatus;
}
if (counter !== 0) {
complete();
}
else {
// last worker clean everything and runs linter in case if there were no errors
deleteTemporaryProjectOutput();
jake.rmRf(taskConfigsFolder);
if (firstErrorStatus === undefined) {
runLinter();
complete();
}
else {
failWithStatus(firstErrorStatus);
}
}
}
});
}
function failWithStatus(status) {
fail("Process exited with code " + status);
}
function finish(errorStatus) {
deleteTemporaryProjectOutput();
if (errorStatus !== undefined) {
failWithStatus(errorStatus);
}
else {
complete();
}
}
function runLinter() {
if (!lintFlag) {
return;
}
var lint = jake.Task['lint'];
lint.addListener('complete', function () {
complete();
});
lint.invoke();
}
}
var testTimeout = 20000;
desc("Runs all the tests in parallel using the built run.js file. Optional arguments are: t[ests]=category1|category2|... d[ebug]=true.");
task("runtests-parallel", ["build-rules", "tests", builtLocalDirectory], function() {
runConsoleTests('min', ['compiler', 'conformance', 'Projects', 'fourslash']);
runConsoleTests('min', /*runInParallel*/ true);
}, {async: true});
desc("Runs the tests using the built run.js file. Optional arguments are: t[ests]=regex r[eporter]=[list|spec|json|<more>] d[ebug]=true color[s]=false lint=true.");
task("runtests", ["build-rules", "tests", builtLocalDirectory], function() {
runConsoleTests('mocha-fivemat-progress-reporter', []);
runConsoleTests('mocha-fivemat-progress-reporter', /*runInParallel*/ false);
}, {async: true});
desc("Generates code coverage data via instanbul");

12
src/harness/compilerRunner.ts
View File

@@ -12,7 +12,7 @@ const enum CompilerTestType {
class CompilerBaselineRunner extends RunnerBase {
private basePath = "tests/cases";
private testSuiteName: string;
private testSuiteName: TestRunnerKind;
private errors: boolean;
private emit: boolean;
private decl: boolean;
@@ -41,6 +41,14 @@ class CompilerBaselineRunner extends RunnerBase {
this.basePath += "/" + this.testSuiteName;
}
public kind() {
return this.testSuiteName;
}
public enumerateTestFiles() {
return this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true });
}
private makeUnitName(name: string, root: string) {
return ts.isRootedDiskPath(name) ? name : ts.combinePaths(root, name);
};
@@ -391,7 +399,7 @@ class CompilerBaselineRunner extends RunnerBase {
// this will set up a series of describe/it blocks to run between the setup and cleanup phases
if (this.tests.length === 0) {
const testFiles = this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true });
const testFiles = this.enumerateTestFiles();
testFiles.forEach(fn => {
fn = fn.replace(/\\/g, "/");
this.checkTestCodeOutput(fn);

12
src/harness/fourslashRunner.ts
View File

@@ -11,7 +11,7 @@ const enum FourSlashTestType {
class FourSlashRunner extends RunnerBase {
protected basePath: string;
protected testSuiteName: string;
protected testSuiteName: TestRunnerKind;
constructor(private testType: FourSlashTestType) {
super();
@@ -35,9 +35,17 @@ class FourSlashRunner extends RunnerBase {
}
}
public enumerateTestFiles() {
return this.enumerateFiles(this.basePath, /\.ts/i, { recursive: false });
}
public kind() {
return this.testSuiteName;
}
public initializeTests() {
if (this.tests.length === 0) {
this.tests = this.enumerateFiles(this.basePath, /\.ts/i, { recursive: false });
this.tests = this.enumerateTestFiles();
}
describe(this.testSuiteName + " tests", () => {

12
src/harness/projectsRunner.ts
View File

@@ -36,11 +36,19 @@ interface BatchCompileProjectTestCaseResult extends CompileProjectFilesResult {
}
class ProjectRunner extends RunnerBase {
public enumerateTestFiles() {
return this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true });
}
public kind(): TestRunnerKind {
return "project";
}
public initializeTests() {
if (this.tests.length === 0) {
const testFiles = this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true });
const testFiles = this.enumerateTestFiles();
testFiles.forEach(fn => {
fn = fn.replace(/\\/g, "/");
this.runProjectTestCase(fn);
});
}

127
src/harness/runner.ts
View File

@@ -31,18 +31,90 @@ function runTests(runners: RunnerBase[]) {
}
}
// users can define tests to run in mytest.config that will override cmd line args, otherwise use cmd line args (test.config), otherwise no options
let mytestconfig = "mytest.config";
let testconfig = "test.config";
let testConfigFile =
Harness.IO.fileExists(mytestconfig) ? Harness.IO.readFile(mytestconfig) :
(Harness.IO.fileExists(testconfig) ? Harness.IO.readFile(testconfig) : "");
function tryGetConfig(args: string[]) {
const prefix = "--config=";
const configPath = ts.forEach(args, arg => arg.lastIndexOf(prefix, 0) === 0 && arg.substr(prefix.length));
// strip leading and trailing quotes from the path (necessary on Windows since shell does not do it automatically)
return configPath && configPath.replace(/(^[\"'])|([\"']$)/g, "");
}
if (testConfigFile !== "") {
const testConfig = JSON.parse(testConfigFile);
function createRunner(kind: TestRunnerKind): RunnerBase {
switch (kind) {
case "conformance":
return new CompilerBaselineRunner(CompilerTestType.Conformance);
case "compiler":
return new CompilerBaselineRunner(CompilerTestType.Regressions);
case "fourslash":
return new FourSlashRunner(FourSlashTestType.Native);
case "fourslash-shims":
return new FourSlashRunner(FourSlashTestType.Shims);
case "fourslash-shims-pp":
return new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess);
case "fourslash-server":
return new FourSlashRunner(FourSlashTestType.Server);
case "project":
return new ProjectRunner();
case "rwc":
return new RWCRunner();
case "test262":
return new Test262BaselineRunner();
}
}
// users can define tests to run in mytest.config that will override cmd line args, otherwise use cmd line args (test.config), otherwise no options
const mytestconfigFileName = "mytest.config";
const testconfigFileName = "test.config";
const customConfig = tryGetConfig(Harness.IO.args());
let testConfigContent =
customConfig && Harness.IO.fileExists(customConfig)
? Harness.IO.readFile(customConfig)
: Harness.IO.fileExists(mytestconfigFileName)
? Harness.IO.readFile(mytestconfigFileName)
: Harness.IO.fileExists(testconfigFileName) ? Harness.IO.readFile(testconfigFileName) : "";
let taskConfigsFolder: string;
let workerCount: number;
let runUnitTests = true;
interface TestConfig {
light?: boolean;
taskConfigsFolder?: string;
workerCount?: number;
tasks?: TaskSet[];
test?: string[];
runUnitTests?: boolean;
}
interface TaskSet {
runner: TestRunnerKind;
files: string[];
}
if (testConfigContent !== "") {
const testConfig = <TestConfig>JSON.parse(testConfigContent);
if (testConfig.light) {
Harness.lightMode = true;
}
if (testConfig.taskConfigsFolder) {
taskConfigsFolder = testConfig.taskConfigsFolder;
}
if (testConfig.runUnitTests !== undefined) {
runUnitTests = testConfig.runUnitTests;
}
if (testConfig.workerCount) {
workerCount = testConfig.workerCount;
}
if (testConfig.tasks) {
for (const taskSet of testConfig.tasks) {
const runner = createRunner(taskSet.runner);
for (const file of taskSet.files) {
runner.addTest(file);
}
runners.push(runner);
}
}
if (testConfig.test && testConfig.test.length > 0) {
for (const option of testConfig.test) {
@@ -106,4 +178,41 @@ if (runners.length === 0) {
// runners.push(new GeneratedFourslashRunner());
}
runTests(runners);
if (taskConfigsFolder) {
// this instance of mocha should only partition work but not run actual tests
runUnitTests = false;
const workerConfigs: TestConfig[] = [];
for (let i = 0; i < workerCount; i++) {
// pass light mode settings to workers
workerConfigs.push({ light: Harness.lightMode, tasks: [] });
}
for (const runner of runners) {
const files = runner.enumerateTestFiles();
const chunkSize = Math.floor(files.length / workerCount) + 1; // add extra 1 to prevent missing tests due to rounding
for (let i = 0; i < workerCount; i++) {
const startPos = i * chunkSize;
const len = Math.min(chunkSize, files.length - startPos);
if (len !== 0) {
workerConfigs[i].tasks.push({
runner: runner.kind(),
files: files.slice(startPos, startPos + len)
});
}
}
}
for (let i = 0; i < workerCount; i++) {
const config = workerConfigs[i];
// use last worker to run unit tests
config.runUnitTests = i === workerCount - 1;
Harness.IO.writeFile(ts.combinePaths(taskConfigsFolder, `task-config${i}.json`), JSON.stringify(workerConfigs[i]));
}
}
else {
runTests(runners);
}
if (!runUnitTests) {
// patch `describe` to skip unit tests
describe = <any>describe.skip;
}

11
src/harness/runnerbase.ts
View File

@@ -1,5 +1,10 @@
/// <reference path="harness.ts" />
type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262";
type CompilerTestKind = "conformance" | "compiler";
type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server";
abstract class RunnerBase {
constructor() { }
@@ -12,9 +17,13 @@ abstract class RunnerBase {
}
public enumerateFiles(folder: string, regex?: RegExp, options?: { recursive: boolean }): string[] {
return Harness.IO.listFiles(Harness.userSpecifiedRoot + folder, regex, { recursive: (options ? options.recursive : false) });
return ts.map(Harness.IO.listFiles(Harness.userSpecifiedRoot + folder, regex, { recursive: (options ? options.recursive : false) }), ts.normalizeSlashes);
}
abstract kind(): TestRunnerKind;
abstract enumerateTestFiles(): string[];
/** Setup the runner's tests so that they are ready to be executed by the harness
* The first test should be a describe/it block that sets up the harness's compiler instance appropriately
*/

10
src/harness/rwcRunner.ts
View File

@@ -224,12 +224,20 @@ namespace RWC {
class RWCRunner extends RunnerBase {
private static sourcePath = "internal/cases/rwc/";
public enumerateTestFiles() {
return Harness.IO.listFiles(RWCRunner.sourcePath, /.+\.json$/);
}
public kind(): TestRunnerKind {
return "rwc";
}
/** Setup the runner's tests so that they are ready to be executed by the harness
* The first test should be a describe/it block that sets up the harness's compiler instance appropriately
*/
public initializeTests(): void {
// Read in and evaluate the test list
const testList = Harness.IO.listFiles(RWCRunner.sourcePath, /.+\.json$/);
const testList = this.enumerateTestFiles();
for (let i = 0; i < testList.length; i++) {
this.runTest(testList[i]);
}

12
src/harness/test262Runner.ts
View File

@@ -98,12 +98,20 @@ class Test262BaselineRunner extends RunnerBase {
});
}
public kind(): TestRunnerKind {
return "test262";
}
public enumerateTestFiles() {
return ts.map(this.enumerateFiles(Test262BaselineRunner.basePath, Test262BaselineRunner.testFileExtensionRegex, { recursive: true }), ts.normalizePath);
}
public initializeTests() {
// this will set up a series of describe/it blocks to run between the setup and cleanup phases
if (this.tests.length === 0) {
const testFiles = this.enumerateFiles(Test262BaselineRunner.basePath, Test262BaselineRunner.testFileExtensionRegex, { recursive: true });
const testFiles = this.enumerateTestFiles();
testFiles.forEach(fn => {
this.runTest(ts.normalizePath(fn));
this.runTest(fn);
});
}
else {