microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2026-03-15 14:05:47 -05:00
Code Issues Packages Projects Releases 212 Wiki Activity

Better-scheduled parallel tests (#18462)

Browse Source 
* Out with the old...

* Brave new world

* Throttle console output

* Batches test messages on large inputs initially

* Move parallel runner code into seperate files
This commit is contained in:
Wesley Wigham
2017-09-14 15:42:06 -07:00
committed by GitHub
parent c522f379b2
commit d1c4754b37
9 changed files with 650 additions and 592 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
Gulpfile.ts
View File

@@ -31,8 +31,6 @@ import merge2 = require("merge2");
import * as os from "os";
import fold = require("travis-fold");
const gulp = helpMaker(originalGulp);
const mochaParallel = require("./scripts/mocha-parallel.js");
const {runTestsInParallel} = mochaParallel;
Error.stackTraceLimit = 1000;
@@ -668,26 +666,9 @@ function runConsoleTests(defaultReporter: string, runInParallel: boolean, done:
}
else {
// run task to load all tests and partition them between workers
const args = [];
args.push("-R", "min");
if (colors) {
args.push("--colors");
}
else {
args.push("--no-colors");
}
args.push(run);
setNodeEnvToDevelopment();
runTestsInParallel(taskConfigsFolder, run, { testTimeout, noColors: colors === " --no-colors " }, function(err) {
// last worker clean everything and runs linter in case if there were no errors
del(taskConfigsFolder).then(() => {
if (!err) {
lintThenFinish();
}
else {
finish(err);
}
});
exec(host, [run], lintThenFinish, function(e, status) {
finish(e, status);
});
}
});
@@ -711,7 +692,7 @@ function runConsoleTests(defaultReporter: string, runInParallel: boolean, done:
function finish(error?: any, errorStatus?: number) {
restoreSavedNodeEnv();
deleteTemporaryProjectOutput().then(() => {
deleteTestConfig().then(deleteTemporaryProjectOutput).then(() => {
if (error !== undefined || errorStatus !== undefined) {
failWithStatus(error, errorStatus);
}
@@ -720,6 +701,10 @@ function runConsoleTests(defaultReporter: string, runInParallel: boolean, done:
}
});
}
function deleteTestConfig() {
return del("test.config");
}
}
gulp.task("runtests-parallel", "Runs all the tests in parallel using the built run.js file. Optional arguments are: --t[ests]=category1|category2|... --d[ebug]=true.", ["build-rules", "tests"], (done) => {
@@ -836,7 +821,7 @@ function cleanTestDirs(done: (e?: any) => void) {
// used to pass data from jake command line directly to run.js
function writeTestConfigFile(tests: string, light: boolean, taskConfigsFolder?: string, workerCount?: number, stackTraceLimit?: string) {
const testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, light, workerCount, stackTraceLimit, taskConfigsFolder });
const testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, light, workerCount, stackTraceLimit, taskConfigsFolder, noColor: !cmdLineOptions["colors"] });
console.log("Running tests with config: " + testConfigContents);
fs.writeFileSync("test.config", testConfigContents);
}

40
Jakefile.js
View File

@@ -1,11 +1,11 @@
// This file contains the build logic for the public repo
// @ts-check
var fs = require("fs");
var os = require("os");
var path = require("path");
var child_process = require("child_process");
var fold = require("travis-fold");
var runTestsInParallel = require("./scripts/mocha-parallel").runTestsInParallel;
var ts = require("./lib/typescript");
@@ -38,7 +38,7 @@ else if (process.env.PATH !== undefined) {
function filesFromConfig(configPath) {
var configText = fs.readFileSync(configPath).toString();
var config = ts.parseConfigFileTextToJson(configPath, configText, /*stripComments*/ true);
var config = ts.parseConfigFileTextToJson(configPath, configText);
if (config.error) {
throw new Error(diagnosticsToString([config.error]));
}
@@ -104,6 +104,9 @@ var harnessCoreSources = [
"loggedIO.ts",
"rwcRunner.ts",
"test262Runner.ts",
"./parallel/shared.ts",
"./parallel/host.ts",
"./parallel/worker.ts",
"runner.ts"
].map(function (f) {
return path.join(harnessDirectory, f);
@@ -596,7 +599,7 @@ file(typesMapOutputPath, function() {
var content = fs.readFileSync(path.join(serverDirectory, 'typesMap.json'));
// Validate that it's valid JSON
try {
JSON.parse(content);
JSON.parse(content.toString());
} catch (e) {
console.log("Parse error in typesMap.json: " + e);
}
@@ -740,7 +743,7 @@ desc("Builds the test infrastructure using the built compiler");
task("tests", ["local", run].concat(libraryTargets));
function exec(cmd, completeHandler, errorHandler) {
var ex = jake.createExec([cmd], { windowsVerbatimArguments: true });
var ex = jake.createExec([cmd], { windowsVerbatimArguments: true, interactive: true });
// Add listeners for output and error
ex.addListener("stdout", function (output) {
process.stdout.write(output);
@@ -783,13 +786,14 @@ function cleanTestDirs() {
}
// used to pass data from jake command line directly to run.js
function writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, stackTraceLimit) {
function writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, stackTraceLimit, colors) {
var testConfigContents = JSON.stringify({
test: tests ? [tests] : undefined,
light: light,
workerCount: workerCount,
taskConfigsFolder: taskConfigsFolder,
stackTraceLimit: stackTraceLimit
stackTraceLimit: stackTraceLimit,
noColor: !colors
});
fs.writeFileSync('test.config', testConfigContents);
}
@@ -831,7 +835,7 @@ function runConsoleTests(defaultReporter, runInParallel) {
}
if (tests || light || taskConfigsFolder) {
writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, stackTraceLimit);
writeTestConfigFile(tests, light, taskConfigsFolder, workerCount, stackTraceLimit, colors);
}
if (tests && tests.toLocaleLowerCase() === "rwc") {
@@ -894,19 +898,15 @@ function runConsoleTests(defaultReporter, runInParallel) {
var savedNodeEnv = process.env.NODE_ENV;
process.env.NODE_ENV = "development";
var startTime = mark();
runTestsInParallel(taskConfigsFolder, run, { testTimeout: testTimeout, noColors: !colors }, function (err) {
exec(host + " " + run, function () {
process.env.NODE_ENV = savedNodeEnv;
measure(startTime);
// last worker clean everything and runs linter in case if there were no errors
deleteTemporaryProjectOutput();
jake.rmRf(taskConfigsFolder);
if (err) {
fail(err);
}
else {
runLinter();
complete();
}
runLinter();
finish();
}, function (e, status) {
process.env.NODE_ENV = savedNodeEnv;
measure(startTime);
finish(status);
});
}
@@ -969,8 +969,8 @@ desc("Runs the tests using the built run.js file like 'jake runtests'. Syntax is
task("runtests-browser", ["browserify", nodeServerOutFile], function () {
cleanTestDirs();
host = "node";
browser = process.env.browser || process.env.b || (os.platform() === "linux" ? "chrome" : "IE");
tests = process.env.test || process.env.tests || process.env.t;
var browser = process.env.browser || process.env.b || (os.platform() === "linux" ? "chrome" : "IE");
var tests = process.env.test || process.env.tests || process.env.t;
var light = process.env.light || false;
var testConfigFile = 'test.config';
if (fs.existsSync(testConfigFile)) {

26
scripts/mocha-none-reporter.js
View File

@@ -1,26 +0,0 @@
/**
* Module dependencies.
*/
var Base = require('mocha').reporters.Base;
/**
* Expose `None`.
*/
exports = module.exports = None;
/**
* Initialize a new `None` test reporter.
*
* @api public
* @param {Runner} runner
*/
function None(runner) {
Base.call(this);
}
/**
* Inherit from `Base.prototype`.
*/
None.prototype.__proto__ = Base.prototype;

405
scripts/mocha-parallel.js
View File

@@ -1,405 +0,0 @@
var tty = require("tty")
, readline = require("readline")
, fs = require("fs")
, path = require("path")
, child_process = require("child_process")
, os = require("os")
, mocha = require("mocha")
, Base = mocha.reporters.Base
, color = Base.color
, cursor = Base.cursor
, ms = require("mocha/lib/ms");
var isatty = tty.isatty(1) && tty.isatty(2);
var tapRangePattern = /^(\d+)\.\.(\d+)(?:$|\r\n?|\n)/;
var tapTestPattern = /^(not\sok|ok)\s+(\d+)\s+(?:-\s+)?(.*)$/;
var tapCommentPattern = /^#(?: (tests|pass|fail) (\d+)$)?/;
exports.runTestsInParallel = runTestsInParallel;
exports.ProgressBars = ProgressBars;
function runTestsInParallel(taskConfigsFolder, run, options, cb) {
if (options === undefined) options = { };
return discoverTests(run, options, function (error) {
if (error) {
return cb(error);
}
return runTests(taskConfigsFolder, run, options, cb);
});
}
function discoverTests(run, options, cb) {
console.log("Discovering tests...");
var cmd = "mocha -R " + require.resolve("./mocha-none-reporter.js") + " " + run;
var p = spawnProcess(cmd);
p.on("exit", function (status) {
if (status) {
cb(new Error("Process exited with code " + status));
}
else {
cb();
}
});
}
function runTests(taskConfigsFolder, run, options, cb) {
var configFiles = fs.readdirSync(taskConfigsFolder);
var numPartitions = configFiles.length;
if (numPartitions <= 0) {
cb();
return;
}
console.log("Running tests on " + numPartitions + " threads...");
var partitions = Array(numPartitions);
var progressBars = new ProgressBars();
progressBars.enable();
var counter = numPartitions;
configFiles.forEach(runTestsInPartition);
function runTestsInPartition(file, index) {
var partition = {
file: path.join(taskConfigsFolder, file),
tests: 0,
passed: 0,
failed: 0,
completed: 0,
current: undefined,
start: undefined,
end: undefined,
catastrophicError: "",
failures: []
};
partitions[index] = partition;
// Set up the progress bar.
updateProgress(0);
// Start the background process.
var cmd = "mocha -t " + (options.testTimeout || 20000) + " -R tap --no-colors " + run + " --config='" + partition.file + "'";
var p = spawnProcess(cmd);
var rl = readline.createInterface({
input: p.stdout,
terminal: false
});
var rlError = readline.createInterface({
input: p.stderr,
terminal: false
});
rl.on("line", onmessage);
rlError.on("line", onErrorMessage);
p.on("exit", onexit)
function onErrorMessage(line) {
partition.catastrophicError += line + os.EOL;
}
function onmessage(line) {
if (partition.start === undefined) {
partition.start = Date.now();
}
var rangeMatch = tapRangePattern.exec(line);
if (rangeMatch) {
partition.tests = parseInt(rangeMatch[2]);
return;
}
var testMatch = tapTestPattern.exec(line);
if (testMatch) {
var test = {
result: testMatch[1],
id: parseInt(testMatch[2]),
name: testMatch[3],
output: []
};
partition.current = test;
partition.completed++;
if (test.result === "ok") {
partition.passed++;
}
else {
partition.failed++;
partition.failures.push(test);
}
var progress = partition.completed / partition.tests;
if (progress < 1) {
updateProgress(progress);
}
return;
}
var commentMatch = tapCommentPattern.exec(line);
if (commentMatch) {
switch (commentMatch[1]) {
case "tests":
partition.current = undefined;
partition.tests = parseInt(commentMatch[2]);
break;
case "pass":
partition.passed = parseInt(commentMatch[2]);
break;
case "fail":
partition.failed = parseInt(commentMatch[2]);
break;
}
return;
}
if (partition.current) {
partition.current.output.push(line);
}
}
function onexit(code) {
if (partition.end === undefined) {
partition.end = Date.now();
}
partition.duration = partition.end - partition.start;
var isPartitionFail = partition.failed || code !== 0;
var summaryColor = isPartitionFail ? "fail" : "green";
var summarySymbol = isPartitionFail ? Base.symbols.err : Base.symbols.ok;
var summaryTests = (isPartitionFail ? partition.passed + "/" + partition.tests : partition.passed) + " passing";
var summaryDuration = "(" + ms(partition.duration) + ")";
var savedUseColors = Base.useColors;
Base.useColors = !options.noColors;
var summary = color(summaryColor, summarySymbol + " " + summaryTests) + " " + color("light", summaryDuration);
Base.useColors = savedUseColors;
updateProgress(1, summary);
signal();
}
function updateProgress(percentComplete, title) {
var progressColor = "pending";
if (partition.failed) {
progressColor = "fail";
}
progressBars.update(
index,
percentComplete,
progressColor,
title
);
}
}
function signal() {
counter--;
if (counter <= 0) {
var reporter = new Base(),
stats = reporter.stats,
failures = reporter.failures;
var duration = 0;
var catastrophicError = "";
for (var i = 0; i < numPartitions; i++) {
var partition = partitions[i];
stats.passes += partition.passed;
stats.failures += partition.failed;
stats.tests += partition.tests;
duration += partition.duration;
if (partition.catastrophicError !== "") {
// Partition is written out to a temporary file as a JSON object.
// Below is an example of how the partition JSON object looks like
// {
// "light":false,
// "tasks":[
// {
// "runner":"compiler",
// "files":["tests/cases/compiler/es6ImportNamedImportParsingError.ts"]
// }
// ],
// "runUnitTests":false
// }
var jsonText = fs.readFileSync(partition.file);
var configObj = JSON.parse(jsonText);
if (configObj.tasks && configObj.tasks[0]) {
catastrophicError += "Error from one or more of these files: " + configObj.tasks[0].files + os.EOL;
catastrophicError += partition.catastrophicError;
catastrophicError += os.EOL;
}
}
for (var j = 0; j < partition.failures.length; j++) {
var failure = partition.failures[j];
failures.push(makeMochaTest(failure));
}
}
stats.duration = duration;
progressBars.disable();
if (options.noColors) {
var savedUseColors = Base.useColors;
Base.useColors = false;
reporter.epilogue();
Base.useColors = savedUseColors;
}
else {
reporter.epilogue();
}
if (catastrophicError !== "") {
return cb(new Error(catastrophicError));
}
if (stats.failures) {
return cb(new Error("Test failures reported: " + stats.failures));
}
else {
return cb();
}
}
}
function makeMochaTest(test) {
return {
fullTitle: function() {
return test.name;
},
err: {
message: test.output[0],
stack: test.output.join(os.EOL)
}
};
}
}
var nodeModulesPathPrefix = path.resolve("./node_modules/.bin/") + path.delimiter;
if (process.env.path !== undefined) {
process.env.path = nodeModulesPathPrefix + process.env.path;
} else if (process.env.PATH !== undefined) {
process.env.PATH = nodeModulesPathPrefix + process.env.PATH;
}
function spawnProcess(cmd, options) {
var shell = process.platform === "win32" ? "cmd" : "/bin/sh";
var prefix = process.platform === "win32" ? "/c" : "-c";
return child_process.spawn(shell, [prefix, cmd], { windowsVerbatimArguments: true });
}
function ProgressBars(options) {
if (!options) options = {};
var open = options.open || '[';
var close = options.close || ']';
var complete = options.complete || '▬';
var incomplete = options.incomplete || Base.symbols.dot;
var maxWidth = Math.floor(Base.window.width * .30) - open.length - close.length - 2;
var width = minMax(options.width || maxWidth, 10, maxWidth);
this._options = {
open: open,
complete: complete,
incomplete: incomplete,
close: close,
width: width
};
this._progressBars = [];
this._lineCount = 0;
this._enabled = false;
}
ProgressBars.prototype = {
enable: function () {
if (!this._enabled) {
process.stdout.write(os.EOL);
this._enabled = true;
}
},
disable: function () {
if (this._enabled) {
process.stdout.write(os.EOL);
this._enabled = false;
}
},
update: function (index, percentComplete, color, title) {
percentComplete = minMax(percentComplete, 0, 1);
var progressBar = this._progressBars[index] || (this._progressBars[index] = { });
var width = this._options.width;
var n = Math.floor(width * percentComplete);
var i = width - n;
if (n === progressBar.lastN && title === progressBar.title && color === progressBar.progressColor) {
return;
}
progressBar.lastN = n;
progressBar.title = title;
progressBar.progressColor = color;
var progress = " ";
progress += this._color('progress', this._options.open);
progress += this._color(color, fill(this._options.complete, n));
progress += this._color('progress', fill(this._options.incomplete, i));
progress += this._color('progress', this._options.close);
if (title) {
progress += this._color('progress', ' ' + title);
}
if (progressBar.text !== progress) {
progressBar.text = progress;
this._render(index);
}
},
_render: function (index) {
if (!this._enabled || !isatty) {
return;
}
cursor.hide();
readline.moveCursor(process.stdout, -process.stdout.columns, -this._lineCount);
var lineCount = 0;
var numProgressBars = this._progressBars.length;
for (var i = 0; i < numProgressBars; i++) {
if (i === index) {
readline.clearLine(process.stdout, 1);
process.stdout.write(this._progressBars[i].text + os.EOL);
}
else {
readline.moveCursor(process.stdout, -process.stdout.columns, +1);
}
lineCount++;
}
this._lineCount = lineCount;
cursor.show();
},
_color: function (type, text) {
return type && !this._options.noColors ? color(type, text) : text;
}
};
function fill(ch, size) {
var s = "";
while (s.length < size) {
s += ch;
}
return s.length > size ? s.substr(0, size) : s;
}
function minMax(value, min, max) {
if (value < min) return min;
if (value > max) return max;
return value;
}

376
src/harness/parallel/host.ts Normal file
View File

@@ -0,0 +1,376 @@
// tslint:disable-next-line
var describe: Mocha.IContextDefinition; // If launched without mocha for parallel mode, we still need a global describe visible to satisfy the parsing of the unit tests
// tslint:disable-next-line
var it: Mocha.ITestDefinition;
namespace Harness.Parallel.Host {
interface ChildProcessPartial {
send(message: any, callback?: (error: Error) => void): boolean;
on(event: "error", listener: (err: Error) => void): this;
on(event: "exit", listener: (code: number, signal: string) => void): this;
on(event: "message", listener: (message: any) => void): this;
disconnect(): void;
}
interface ProgressBarsOptions {
open: string;
close: string;
complete: string;
incomplete: string;
width: number;
noColors: boolean;
}
interface ProgressBar {
lastN?: number;
title?: string;
progressColor?: string;
text?: string;
}
export function start() {
console.log("Discovering tests...");
const discoverStart = +(new Date());
const { statSync }: { statSync(path: string): { size: number }; } = require("fs");
const tasks: { runner: TestRunnerKind, file: string, size: number }[] = [];
let totalSize = 0;
for (const runner of runners) {
const files = runner.enumerateTestFiles();
for (const file of files) {
const size = statSync(file).size;
tasks.push({ runner: runner.kind(), file, size });
totalSize += size;
}
}
tasks.sort((a, b) => a.size - b.size);
const batchSize = (totalSize / workerCount) * 0.9;
console.log(`Discovered ${tasks.length} test files in ${+(new Date()) - discoverStart}ms.`);
console.log(`Starting to run tests using ${workerCount} threads...`);
const { fork }: { fork(modulePath: string, args?: string[], options?: {}): ChildProcessPartial; } = require("child_process");
const totalFiles = tasks.length;
let passingFiles = 0;
let failingFiles = 0;
let errorResults: ErrorInfo[] = [];
let totalPassing = 0;
const startTime = Date.now();
const progressBars = new ProgressBars({ noColors });
const progressUpdateInterval = 1 / progressBars._options.width;
let nextProgress = progressUpdateInterval;
const workers: ChildProcessPartial[] = [];
for (let i = 0; i < workerCount; i++) {
// TODO: Just send the config over the IPC channel or in the command line arguments
const config: TestConfig = { light: Harness.lightMode, listenForWork: true, runUnitTests: runners.length === 1 ? false : i === workerCount - 1 };
const configPath = ts.combinePaths(taskConfigsFolder, `task-config${i}.json`);
Harness.IO.writeFile(configPath, JSON.stringify(config));
const child = fork(__filename, [`--config="${configPath}"`]);
child.on("error", err => {
child.disconnect();
console.error("Unexpected error in child process:");
console.error(err);
return process.exit(2);
});
child.on("exit", (code, _signal) => {
if (code !== 0) {
console.error("Test worker process exited with nonzero exit code!");
return process.exit(2);
}
});
child.on("message", (data: ParallelClientMessage) => {
switch (data.type) {
case "error": {
child.disconnect();
console.error(`Test worker encounted unexpected error and was forced to close:
Message: ${data.payload.error}
Stack: ${data.payload.stack}`);
return process.exit(2);
}
case "progress":
case "result": {
totalPassing += data.payload.passing;
if (data.payload.errors.length) {
errorResults = errorResults.concat(data.payload.errors);
failingFiles++;
}
else {
passingFiles++;
}
const progress = (failingFiles + passingFiles) / totalFiles;
if (progress >= nextProgress) {
while (nextProgress < progress) {
nextProgress += progressUpdateInterval;
}
updateProgress(progress, errorResults.length ? `${errorResults.length} failing` : `${totalPassing} passing`, errorResults.length ? "fail" : undefined);
}
if (failingFiles + passingFiles === totalFiles) {
// Done. Finished every task and collected results.
child.send({ type: "close" });
child.disconnect();
return outputFinalResult();
}
if (tasks.length === 0) {
// No more tasks to distribute
child.send({ type: "close" });
child.disconnect();
return;
}
if (data.type === "result") {
child.send({ type: "test", payload: tasks.pop() });
}
}
}
});
workers.push(child);
}
// It's only really worth doing an initial batching if there are a ton of files to go through
if (totalFiles > 1000) {
console.log("Batching initial test lists...");
const batches: { runner: TestRunnerKind, file: string, size: number }[][] = new Array(workerCount);
const doneBatching = new Array(workerCount);
batcher: while (true) {
for (let i = 0; i < workerCount; i++) {
if (tasks.length === 0) {
// TODO: This indicates a particularly suboptimal packing
break batcher;
}
if (doneBatching[i]) {
continue;
}
if (!batches[i]) {
batches[i] = [];
}
const total = batches[i].reduce((p, c) => p + c.size, 0);
if (total >= batchSize && !doneBatching[i]) {
doneBatching[i] = true;
continue;
}
batches[i].push(tasks.pop());
}
for (let j = 0; j < workerCount; j++) {
if (!doneBatching[j]) {
continue;
}
}
break;
}
console.log(`Batched into ${workerCount} groups with approximate total file sizes of ${Math.floor(batchSize)} bytes in each group.`);
for (const worker of workers) {
const action: ParallelBatchMessage = { type: "batch", payload: batches.pop() };
if (!action.payload[0]) {
throw new Error(`Tried to send invalid message ${action}`);
}
worker.send(action);
}
}
else {
for (let i = 0; i < workerCount; i++) {
workers[i].send({ type: "test", payload: tasks.pop() });
}
}
progressBars.enable();
updateProgress(0);
let duration: number;
const ms = require("mocha/lib/ms");
function completeBar() {
const isPartitionFail = failingFiles !== 0;
const summaryColor = isPartitionFail ? "fail" : "green";
const summarySymbol = isPartitionFail ? Base.symbols.err : Base.symbols.ok;
const summaryTests = (isPartitionFail ? totalPassing + "/" + (errorResults.length + totalPassing) : totalPassing) + " passing";
const summaryDuration = "(" + ms(duration) + ")";
const savedUseColors = Base.useColors;
Base.useColors = !noColors;
const summary = color(summaryColor, summarySymbol + " " + summaryTests) + " " + color("light", summaryDuration);
Base.useColors = savedUseColors;
updateProgress(1, summary);
}
function updateProgress(percentComplete: number, title?: string, titleColor?: string) {
let progressColor = "pending";
if (failingFiles) {
progressColor = "fail";
}
progressBars.update(
0,
percentComplete,
progressColor,
title,
titleColor
);
}
function outputFinalResult() {
duration = Date.now() - startTime;
completeBar();
progressBars.disable();
const reporter = new Base();
const stats = reporter.stats;
const failures = reporter.failures;
stats.passes = totalPassing;
stats.failures = errorResults.length;
stats.tests = totalPassing + errorResults.length;
stats.duration = duration;
for (let j = 0; j < errorResults.length; j++) {
const failure = errorResults[j];
failures.push(makeMochaTest(failure));
}
if (noColors) {
const savedUseColors = Base.useColors;
Base.useColors = false;
reporter.epilogue();
Base.useColors = savedUseColors;
}
else {
reporter.epilogue();
}
process.exit(errorResults.length);
}
function makeMochaTest(test: ErrorInfo) {
return {
fullTitle: () => {
return test.name;
},
err: {
message: test.error,
stack: test.stack
}
};
}
describe = ts.noop as any; // Disable unit tests
return;
}
const Mocha = require("mocha");
const Base = Mocha.reporters.Base;
const color = Base.color;
const cursor = Base.cursor;
const readline = require("readline");
const os = require("os");
const tty: { isatty(x: number): boolean } = require("tty");
const isatty = tty.isatty(1) && tty.isatty(2);
class ProgressBars {
public readonly _options: Readonly<ProgressBarsOptions>;
private _enabled: boolean;
private _lineCount: number;
private _progressBars: ProgressBar[];
constructor(options?: Partial<ProgressBarsOptions>) {
if (!options) options = {};
const open = options.open || "[";
const close = options.close || "]";
const complete = options.complete || "▬";
const incomplete = options.incomplete || Base.symbols.dot;
const maxWidth = Base.window.width - open.length - close.length - 30;
const width = minMax(options.width || maxWidth, 10, maxWidth);
this._options = {
open,
complete,
incomplete,
close,
width,
noColors: options.noColors || false
};
this._progressBars = [];
this._lineCount = 0;
this._enabled = false;
}
enable() {
if (!this._enabled) {
process.stdout.write(os.EOL);
this._enabled = true;
}
}
disable() {
if (this._enabled) {
process.stdout.write(os.EOL);
this._enabled = false;
}
}
update(index: number, percentComplete: number, color: string, title: string, titleColor?: string) {
percentComplete = minMax(percentComplete, 0, 1);
const progressBar = this._progressBars[index] || (this._progressBars[index] = { });
const width = this._options.width;
const n = Math.floor(width * percentComplete);
const i = width - n;
if (n === progressBar.lastN && title === progressBar.title && color === progressBar.progressColor) {
return;
}
progressBar.lastN = n;
progressBar.title = title;
progressBar.progressColor = color;
let progress = " ";
progress += this._color("progress", this._options.open);
progress += this._color(color, fill(this._options.complete, n));
progress += this._color("progress", fill(this._options.incomplete, i));
progress += this._color("progress", this._options.close);
if (title) {
progress += this._color(titleColor || "progress", " " + title);
}
if (progressBar.text !== progress) {
progressBar.text = progress;
this._render(index);
}
}
private _render(index: number) {
if (!this._enabled || !isatty) {
return;
}
cursor.hide();
readline.moveCursor(process.stdout, -process.stdout.columns, -this._lineCount);
let lineCount = 0;
const numProgressBars = this._progressBars.length;
for (let i = 0; i < numProgressBars; i++) {
if (i === index) {
readline.clearLine(process.stdout, 1);
process.stdout.write(this._progressBars[i].text + os.EOL);
}
else {
readline.moveCursor(process.stdout, -process.stdout.columns, +1);
}
lineCount++;
}
this._lineCount = lineCount;
cursor.show();
}
private _color(type: string, text: string) {
return type && !this._options.noColors ? color(type, text) : text;
}
}
function fill(ch: string, size: number) {
let s = "";
while (s.length < size) {
s += ch;
}
return s.length > size ? s.substr(0, size) : s;
}
function minMax(value: number, min: number, max: number) {
if (value < min) return min;
if (value > max) return max;
return value;
}
}

14
src/harness/parallel/shared.ts Normal file
View File

@@ -0,0 +1,14 @@
/// <reference path="./host.ts" />
/// <reference path="./worker.ts" />
namespace Harness.Parallel {
export type ParallelTestMessage = { type: "test", payload: { runner: TestRunnerKind, file: string } } | never;
export type ParallelBatchMessage = { type: "batch", payload: ParallelTestMessage["payload"][] } | never;
export type ParallelCloseMessage = { type: "close" } | never;
export type ParallelHostMessage = ParallelTestMessage | ParallelCloseMessage | ParallelBatchMessage;
export type ParallelErrorMessage = { type: "error", payload: { error: string, stack: string } } | never;
export type ErrorInfo = ParallelErrorMessage["payload"] & { name: string };
export type ParallelResultMessage = { type: "result", payload: { passing: number, errors: ErrorInfo[] } } | never;
export type ParallelBatchProgressMessage = { type: "progress", payload: ParallelResultMessage["payload"] } | never;
export type ParallelClientMessage = ParallelErrorMessage | ParallelResultMessage | ParallelBatchProgressMessage;
}

123
src/harness/parallel/worker.ts Normal file
View File

@@ -0,0 +1,123 @@
namespace Harness.Parallel.Worker {
let errors: ErrorInfo[] = [];
let passing = 0;
function resetShimHarnessAndExecute(runner: RunnerBase) {
errors = [];
passing = 0;
runner.initializeTests();
return { errors, passing };
}
function shimMochaHarness() {
(global as any).before = undefined;
(global as any).after = undefined;
(global as any).beforeEach = undefined;
let beforeEachFunc: Function;
describe = ((_name, callback) => {
const fakeContext: Mocha.ISuiteCallbackContext = {
retries() { return this; },
slow() { return this; },
timeout() { return this; },
};
(before as any) = (cb: Function) => cb();
let afterFunc: Function;
(after as any) = (cb: Function) => afterFunc = cb;
const savedBeforeEach = beforeEachFunc;
(beforeEach as any) = (cb: Function) => beforeEachFunc = cb;
callback.call(fakeContext);
afterFunc && afterFunc();
afterFunc = undefined;
beforeEachFunc = savedBeforeEach;
}) as Mocha.IContextDefinition;
it = ((name, callback) => {
const fakeContext: Mocha.ITestCallbackContext = {
skip() { return this; },
timeout() { return this; },
retries() { return this; },
slow() { return this; },
};
// TODO: If we ever start using async test completions, polyfill the `done` parameter/promise return handling
if (beforeEachFunc) {
try {
beforeEachFunc();
}
catch (error) {
errors.push({ error: error.message, stack: error.stack, name });
return;
}
}
try {
callback.call(fakeContext);
}
catch (error) {
errors.push({ error: error.message, stack: error.stack, name });
return;
}
passing++;
}) as Mocha.ITestDefinition;
}
export function start() {
let initialized = false;
const runners = ts.createMap<RunnerBase>();
process.on("message", (data: ParallelHostMessage) => {
if (!initialized) {
initialized = true;
shimMochaHarness();
}
switch (data.type) {
case "test":
const { runner, file } = data.payload;
if (!runner) {
console.error(data);
}
const message: ParallelResultMessage = { type: "result", payload: handleTest(runner, file) };
process.send(message);
break;
case "close":
process.exit(0);
break;
case "batch": {
const items = data.payload;
for (let i = 0; i < items.length; i++) {
const { runner, file } = items[i];
if (!runner) {
console.error(data);
}
let message: ParallelBatchProgressMessage | ParallelResultMessage;
const payload = handleTest(runner, file);
if (i === (items.length - 1)) {
message = { type: "result", payload };
}
else {
message = { type: "progress", payload };
}
process.send(message);
}
break;
}
}
});
process.on("uncaughtException", error => {
const message: ParallelErrorMessage = { type: "error", payload: { error: error.message, stack: error.stack } };
process.send(message);
});
if (!runUnitTests) {
// ensure unit tests do not get run
describe = ts.noop as any;
}
else {
initialized = true;
shimMochaHarness();
}
function handleTest(runner: TestRunnerKind, file: string) {
if (!runners.has(runner)) {
runners.set(runner, createRunner(runner));
}
const instance = runners.get(runner);
instance.tests = [file];
return resetShimHarnessAndExecute(instance);
}
}
}

224
src/harness/runner.ts
View File

@@ -19,6 +19,7 @@
/// <reference path="projectsRunner.ts" />
/// <reference path="rwcRunner.ts" />
/// <reference path="harness.ts" />
/// <reference path="./parallel/shared.ts" />
let runners: RunnerBase[] = [];
let iterations = 1;
@@ -59,6 +60,7 @@ function createRunner(kind: TestRunnerKind): RunnerBase {
case "test262":
return new Test262BaselineRunner();
}
ts.Debug.fail(`Unknown runner kind ${kind}`);
}
if (Harness.IO.tryEnableSourceMapsForHost && /^development$/i.test(Harness.IO.getEnvironmentVariable("NODE_ENV"))) {
@@ -81,15 +83,17 @@ let testConfigContent =
let taskConfigsFolder: string;
let workerCount: number;
let runUnitTests = true;
let noColors = false;
interface TestConfig {
light?: boolean;
taskConfigsFolder?: string;
listenForWork?: boolean;
workerCount?: number;
stackTraceLimit?: number | "full";
tasks?: TaskSet[];
test?: string[];
runUnitTests?: boolean;
noColors?: boolean;
}
interface TaskSet {
@@ -97,138 +101,122 @@ interface TaskSet {
files: string[];
}
if (testConfigContent !== "") {
const testConfig = <TestConfig>JSON.parse(testConfigContent);
if (testConfig.light) {
Harness.lightMode = true;
}
if (testConfig.taskConfigsFolder) {
taskConfigsFolder = testConfig.taskConfigsFolder;
}
if (testConfig.runUnitTests !== undefined) {
runUnitTests = testConfig.runUnitTests;
}
if (testConfig.workerCount) {
workerCount = testConfig.workerCount;
}
if (testConfig.tasks) {
for (const taskSet of testConfig.tasks) {
const runner = createRunner(taskSet.runner);
for (const file of taskSet.files) {
runner.addTest(file);
}
runners.push(runner);
function handleTestConfig() {
if (testConfigContent !== "") {
const testConfig = <TestConfig>JSON.parse(testConfigContent);
if (testConfig.light) {
Harness.lightMode = true;
}
}
if (testConfig.stackTraceLimit === "full") {
(<any>Error).stackTraceLimit = Infinity;
}
else if ((+testConfig.stackTraceLimit | 0) > 0) {
(<any>Error).stackTraceLimit = testConfig.stackTraceLimit;
}
if (testConfig.test && testConfig.test.length > 0) {
for (const option of testConfig.test) {
if (!option) {
continue;
}
switch (option) {
case "compiler":
runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance));
runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions));
runners.push(new ProjectRunner());
break;
case "conformance":
runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance));
break;
case "project":
runners.push(new ProjectRunner());
break;
case "fourslash":
runners.push(new FourSlashRunner(FourSlashTestType.Native));
break;
case "fourslash-shims":
runners.push(new FourSlashRunner(FourSlashTestType.Shims));
break;
case "fourslash-shims-pp":
runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess));
break;
case "fourslash-server":
runners.push(new FourSlashRunner(FourSlashTestType.Server));
break;
case "fourslash-generated":
runners.push(new GeneratedFourslashRunner(FourSlashTestType.Native));
break;
case "rwc":
runners.push(new RWCRunner());
break;
case "test262":
runners.push(new Test262BaselineRunner());
break;
}
if (testConfig.runUnitTests !== undefined) {
runUnitTests = testConfig.runUnitTests;
}
if (testConfig.workerCount) {
workerCount = +testConfig.workerCount;
}
if (testConfig.taskConfigsFolder) {
taskConfigsFolder = testConfig.taskConfigsFolder;
}
if (testConfig.noColors !== undefined) {
noColors = testConfig.noColors;
}
}
}
if (runners.length === 0) {
// compiler
runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance));
runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions));
if (testConfig.stackTraceLimit === "full") {
(<any>Error).stackTraceLimit = Infinity;
}
else if ((+testConfig.stackTraceLimit | 0) > 0) {
(<any>Error).stackTraceLimit = testConfig.stackTraceLimit;
}
if (testConfig.listenForWork) {
return true;
}
// TODO: project tests don't work in the browser yet
if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) {
runners.push(new ProjectRunner());
}
if (testConfig.test && testConfig.test.length > 0) {
for (const option of testConfig.test) {
if (!option) {
continue;
}
// language services
runners.push(new FourSlashRunner(FourSlashTestType.Native));
runners.push(new FourSlashRunner(FourSlashTestType.Shims));
runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess));
runners.push(new FourSlashRunner(FourSlashTestType.Server));
// runners.push(new GeneratedFourslashRunner());
}
if (taskConfigsFolder) {
// this instance of mocha should only partition work but not run actual tests
runUnitTests = false;
const workerConfigs: TestConfig[] = [];
for (let i = 0; i < workerCount; i++) {
// pass light mode settings to workers
workerConfigs.push({ light: Harness.lightMode, tasks: [] });
}
for (const runner of runners) {
const files = runner.enumerateTestFiles();
const chunkSize = Math.floor(files.length / workerCount) + 1; // add extra 1 to prevent missing tests due to rounding
for (let i = 0; i < workerCount; i++) {
const startPos = i * chunkSize;
const len = Math.min(chunkSize, files.length - startPos);
if (len > 0) {
workerConfigs[i].tasks.push({
runner: runner.kind(),
files: files.slice(startPos, startPos + len)
});
switch (option) {
case "compiler":
runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance));
runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions));
runners.push(new ProjectRunner());
break;
case "conformance":
runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance));
break;
case "project":
runners.push(new ProjectRunner());
break;
case "fourslash":
runners.push(new FourSlashRunner(FourSlashTestType.Native));
break;
case "fourslash-shims":
runners.push(new FourSlashRunner(FourSlashTestType.Shims));
break;
case "fourslash-shims-pp":
runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess));
break;
case "fourslash-server":
runners.push(new FourSlashRunner(FourSlashTestType.Server));
break;
case "fourslash-generated":
runners.push(new GeneratedFourslashRunner(FourSlashTestType.Native));
break;
case "rwc":
runners.push(new RWCRunner());
break;
case "test262":
runners.push(new Test262BaselineRunner());
break;
}
}
}
}
for (let i = 0; i < workerCount; i++) {
const config = workerConfigs[i];
// use last worker to run unit tests if we're not just running a single specific runner
config.runUnitTests = runners.length !== 1 && i === workerCount - 1;
Harness.IO.writeFile(ts.combinePaths(taskConfigsFolder, `task-config${i}.json`), JSON.stringify(workerConfigs[i]));
if (runners.length === 0) {
// compiler
runners.push(new CompilerBaselineRunner(CompilerTestType.Conformance));
runners.push(new CompilerBaselineRunner(CompilerTestType.Regressions));
// TODO: project tests don"t work in the browser yet
if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) {
runners.push(new ProjectRunner());
}
// language services
runners.push(new FourSlashRunner(FourSlashTestType.Native));
runners.push(new FourSlashRunner(FourSlashTestType.Shims));
runners.push(new FourSlashRunner(FourSlashTestType.ShimsWithPreprocess));
runners.push(new FourSlashRunner(FourSlashTestType.Server));
// runners.push(new GeneratedFourslashRunner());
}
}
else {
function beginTests() {
if (ts.Debug.isDebugging) {
ts.Debug.enableDebugInfo();
}
runTests(runners);
if (!runUnitTests) {
// patch `describe` to skip unit tests
describe = ts.noop as any;
}
}
if (!runUnitTests) {
// patch `describe` to skip unit tests
describe = ts.noop as any;
function startTestEnvironment() {
const isWorker = handleTestConfig();
if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) {
if (isWorker) {
return Harness.Parallel.Worker.start();
}
else if (taskConfigsFolder && workerCount && workerCount > 1) {
return Harness.Parallel.Host.start();
}
}
beginTests();
}
startTestEnvironment();

3
src/harness/tsconfig.json
View File

@@ -92,6 +92,9 @@
"loggedIO.ts",
"rwcRunner.ts",
"test262Runner.ts",
"./parallel/shared.ts",
"./parallel/host.ts",
"./parallel/worker.ts",
"runner.ts",
"../server/protocol.ts",
"../server/session.ts",