diff --git a/.gitignore b/.gitignore index b5aecdbe35d..fdcea6a6800 100644 --- a/.gitignore +++ b/.gitignore @@ -73,4 +73,5 @@ tests/cases/user/*/**/*.d.ts !tests/cases/user/zone.js/ !tests/cases/user/bignumber.js/ !tests/cases/user/discord.js/ -tests/baselines/reference/dt \ No newline at end of file +tests/baselines/reference/dt +.failed-tests \ No newline at end of file diff --git a/Gulpfile.js b/Gulpfile.js index db7bb76d34b..34808bd10bb 100644 --- a/Gulpfile.js +++ b/Gulpfile.js @@ -38,7 +38,7 @@ const constEnumCaptureRegexp = /^(\s*)(export )?const enum (\S+) {(\s*)$/gm; const constEnumReplacement = "$1$2enum $3 {$4"; const cmdLineOptions = minimist(process.argv.slice(2), { - boolean: ["debug", "inspect", "light", "colors", "lint", "soft", "fix"], + boolean: ["debug", "inspect", "light", "colors", "lint", "soft", "fix", "failed", "keepFailed"], string: ["browser", "tests", "host", "reporter", "stackTraceLimit", "timeout"], alias: { "b": "browser", @@ -598,7 +598,6 @@ gulp.task("LKG", "Makes a new LKG out of the built js files", ["clean", "dontUse return seq; }); - // Task to build the tests infrastructure using the built compiler const run = path.join(builtLocalDirectory, "run.js"); gulp.task(run, /*help*/ false, [servicesFile, tsserverLibraryFile], () => { @@ -653,10 +652,12 @@ function runConsoleTests(defaultReporter, runInParallel, done) { let testTimeout = cmdLineOptions.timeout; const debug = cmdLineOptions.debug; const inspect = cmdLineOptions.inspect; - const tests = cmdLineOptions.tests; + let tests = cmdLineOptions.tests; const runners = cmdLineOptions.runners; const light = cmdLineOptions.light; const stackTraceLimit = cmdLineOptions.stackTraceLimit; + const failed = cmdLineOptions.failed; + const keepFailed = cmdLineOptions.keepFailed || failed; const testConfigFile = "test.config"; if (fs.existsSync(testConfigFile)) { fs.unlinkSync(testConfigFile); @@ -679,8 +680,8 @@ function runConsoleTests(defaultReporter, runInParallel, done) { testTimeout = 400000; } - if (tests || runners || light || testTimeout || taskConfigsFolder) { - writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout); + if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) { + writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed); } const colors = cmdLineOptions.colors; @@ -690,7 +691,8 @@ function runConsoleTests(defaultReporter, runInParallel, done) { // default timeout is 2sec which really should be enough, but maybe we just need a small amount longer if (!runInParallel) { const args = []; - args.push("-R", reporter); + args.push("-R", "scripts/failed-tests"); + args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"'); if (tests) { args.push("-g", `"${tests}"`); } @@ -711,8 +713,12 @@ function runConsoleTests(defaultReporter, runInParallel, done) { } args.push(run); setNodeEnvToDevelopment(); - exec(mocha, args, lintThenFinish, finish); - + if (failed) { + exec(host, ["scripts/run-failed-tests.js"].concat(args), lintThenFinish, finish); + } + else { + exec(mocha, args, lintThenFinish, finish); + } } else { // run task to load all tests and partition them between workers @@ -888,8 +894,9 @@ function cleanTestDirs(done) { * @param {number=} workerCount * @param {string=} stackTraceLimit * @param {number=} timeout + * @param {boolean=} keepFailed */ -function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout) { +function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed) { const testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, runner: runners ? runners.split(",") : undefined, @@ -899,6 +906,7 @@ function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCou taskConfigsFolder, noColor: !cmdLineOptions.colors, timeout, + keepFailed }); console.log("Running tests with config: " + testConfigContents); fs.writeFileSync("test.config", testConfigContents); diff --git a/Jakefile.js b/Jakefile.js index 9b3876ea13f..3a0aee0ab02 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -388,6 +388,8 @@ function runConsoleTests(defaultReporter, runInParallel) { const runners = process.env.runners || process.env.runner || process.env.ru; const tests = process.env.test || process.env.tests || process.env.t; const light = process.env.light === undefined || process.env.light !== "false"; + const failed = process.env.failed; + const keepFailed = process.env.keepFailed || failed; const stackTraceLimit = process.env.stackTraceLimit; const colorsFlag = process.env.color || process.env.colors; const colors = colorsFlag !== "false" && colorsFlag !== "0"; @@ -418,8 +420,8 @@ function runConsoleTests(defaultReporter, runInParallel) { testTimeout = 800000; } - if (tests || runners || light || testTimeout || taskConfigsFolder) { - writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout); + if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) { + writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout, keepFailed); } // timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally @@ -427,7 +429,8 @@ function runConsoleTests(defaultReporter, runInParallel) { if (!runInParallel) { var startTime = Travis.mark(); var args = []; - args.push("-R", reporter); + args.push("-R", "scripts/failed-tests"); + args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"'); if (tests) args.push("-g", `"${tests}"`); args.push(colors ? "--colors" : "--no-colors"); if (bail) args.push("--bail"); @@ -438,7 +441,14 @@ function runConsoleTests(defaultReporter, runInParallel) { } args.push(Paths.builtLocalRun); - var cmd = "mocha " + args.join(" "); + var cmd; + if (failed) { + args.unshift("scripts/run-failed-tests.js"); + cmd = host + " " + args.join(" "); + } + else { + cmd = "mocha " + args.join(" "); + } var savedNodeEnv = process.env.NODE_ENV; process.env.NODE_ENV = "development"; exec(cmd, function () { @@ -499,7 +509,7 @@ function runConsoleTests(defaultReporter, runInParallel) { } // used to pass data from jake command line directly to run.js -function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout) { +function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout, keepFailed) { var testConfigContents = JSON.stringify({ runners: runners ? runners.split(",") : undefined, test: tests ? [tests] : undefined, @@ -508,7 +518,8 @@ function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCou taskConfigsFolder: taskConfigsFolder, stackTraceLimit: stackTraceLimit, noColor: !colors, - timeout: testTimeout + timeout: testTimeout, + keepFailed: keepFailed }); fs.writeFileSync('test.config', testConfigContents, { encoding: "utf-8" }); } diff --git a/scripts/failed-tests.d.ts b/scripts/failed-tests.d.ts new file mode 100644 index 00000000000..e82c0b178e8 --- /dev/null +++ b/scripts/failed-tests.d.ts @@ -0,0 +1,22 @@ +import Mocha = require("mocha"); + +export = FailedTestsReporter; + +declare class FailedTestsReporter extends Mocha.reporters.Base { + passes: Mocha.Test[]; + failures: Mocha.Test[]; + reporterOptions: FailedTestsReporter.ReporterOptions; + reporter?: Mocha.reporters.Base; + constructor(runner: Mocha.Runner, options?: { reporterOptions?: FailedTestsReporter.ReporterOptions }); + static writeFailures(file: string, passes: ReadonlyArray, failures: ReadonlyArray, keepFailed: boolean, done: (err?: NodeJS.ErrnoException) => void): void; + done(failures: number, fn?: (failures: number) => void): void; +} + +declare namespace FailedTestsReporter { + interface ReporterOptions { + file?: string; + keepFailed?: boolean; + reporter?: string | Mocha.ReporterConstructor; + reporterOptions?: any; + } +} \ No newline at end of file diff --git a/scripts/failed-tests.js b/scripts/failed-tests.js new file mode 100644 index 00000000000..ca26c432948 --- /dev/null +++ b/scripts/failed-tests.js @@ -0,0 +1,117 @@ +// @ts-check +const Mocha = require("mocha"); +const path = require("path"); +const fs = require("fs"); +const os = require("os"); + +/** + * .failed-tests reporter + * + * @typedef {Object} ReporterOptions + * @property {string} [file] + * @property {boolean} [keepFailed] + * @property {string|Mocha.ReporterConstructor} [reporter] + * @property {*} [reporterOptions] + */ +class FailedTestsReporter extends Mocha.reporters.Base { + /** + * @param {Mocha.Runner} runner + * @param {{ reporterOptions?: ReporterOptions }} [options] + */ + constructor(runner, options) { + super(runner, options); + if (!runner) return; + + const reporterOptions = this.reporterOptions = options.reporterOptions || {}; + if (reporterOptions.file === undefined) reporterOptions.file = ".failed-tests"; + if (reporterOptions.keepFailed === undefined) reporterOptions.keepFailed = false; + if (reporterOptions.reporter) { + /** @type {Mocha.ReporterConstructor} */ + let reporter; + if (typeof reporterOptions.reporter === "function") { + reporter = reporterOptions.reporter; + } + else if (Mocha.reporters[reporterOptions.reporter]) { + reporter = Mocha.reporters[reporterOptions.reporter]; + } + else { + try { + reporter = require(reporterOptions.reporter); + } + catch (_) { + reporter = require(path.resolve(process.cwd(), reporterOptions.reporter)); + } + } + + const newOptions = Object.assign({}, options, { reporterOptions: reporterOptions.reporterOptions || {} }); + this.reporter = new reporter(runner, newOptions); + } + + /** @type {Mocha.Test[]} */ + this.passes = []; + + /** @type {Mocha.Test[]} */ + this.failures = []; + + runner.on("pass", test => this.passes.push(test)); + runner.on("fail", test => this.failures.push(test)); + } + + /** + * @param {string} file + * @param {ReadonlyArray} passes + * @param {ReadonlyArray} failures + * @param {boolean} keepFailed + * @param {(err?: NodeJS.ErrnoException) => void} done + */ + static writeFailures(file, passes, failures, keepFailed, done) { + const failingTests = new Set(fs.existsSync(file) ? readTests() : undefined); + if (failingTests.size > 0) { + for (const test of passes) { + const title = test.fullTitle().trim(); + if (title) failingTests.delete(title); + } + } + for (const test of failures) { + const title = test.fullTitle().trim(); + if (title) failingTests.add(title); + } + if (failingTests.size > 0) { + const failed = Array.from(failingTests).join(os.EOL); + fs.writeFile(file, failed, "utf8", done); + } + else if (!keepFailed) { + fs.unlink(file, done); + } + else { + done(); + } + + function readTests() { + return fs.readFileSync(file, "utf8") + .split(/\r?\n/g) + .map(line => line.trim()) + .filter(line => line.length > 0); + } + } + + /** + * @param {number} failures + * @param {(failures: number) => void} [fn] + */ + done(failures, fn) { + FailedTestsReporter.writeFailures(this.reporterOptions.file, this.passes, this.failures, this.reporterOptions.keepFailed || this.stats.tests === 0, (err) => { + const reporter = this.reporter; + if (reporter && reporter.done) { + reporter.done(failures, fn); + } + else if (fn) { + fn(failures); + } + + if (err) console.error(err); + }); + } +} + +module.exports = FailedTestsReporter; \ No newline at end of file diff --git a/scripts/run-failed-tests.js b/scripts/run-failed-tests.js new file mode 100644 index 00000000000..a716434d1d6 --- /dev/null +++ b/scripts/run-failed-tests.js @@ -0,0 +1,93 @@ +const spawn = require('child_process').spawn; +const os = require("os"); +const fs = require("fs"); +const path = require("path"); + +let grep; +try { + const failedTests = fs.readFileSync(".failed-tests", "utf8"); + grep = failedTests + .split(/\r?\n/g) + .map(test => test.trim()) + .filter(test => test.length > 0) + .map(escapeRegExp); +} +catch (e) { + grep = []; +} + +let args = []; +let waitForGrepValue = false; +let grepIndex = -1; +process.argv.slice(2).forEach((arg, index) => { + const [flag, value] = arg.split('='); + if (flag === "g" || flag === "grep") { + grepIndex = index - 1; + waitForGrepValue = arg !== flag; + if (!waitForGrepValue) grep.push(value.replace(/^"|"$/g, "")); + return; + } + if (waitForGrepValue) { + grep.push(arg.replace(/^"|"$/g, "")); + waitForGrepValue = false; + return; + } + args.push(arg); +}); + +let mocha = "./node_modules/mocha/bin/mocha"; +let grepOption; +let grepOptionValue; +let grepFile; +if (grep.length) { + grepOption = "--grep"; + grepOptionValue = grep.join("|"); + if (grepOptionValue.length > 20) { + grepFile = path.resolve(os.tmpdir(), ".failed-tests.opts"); + fs.writeFileSync(grepFile, `--grep ${grepOptionValue}`, "utf8"); + grepOption = "--opts"; + grepOptionValue = grepFile; + mocha = "./node_modules/mocha/bin/_mocha"; + } +} + +if (grepOption) { + if (grepIndex >= 0) { + args.splice(grepIndex, 0, grepOption, grepOptionValue); + } + else { + args.push(grepOption, grepOptionValue); + } +} + +args.unshift(path.resolve(mocha)); + +console.log(args.join(" ")); +const proc = spawn(process.execPath, args, { + stdio: 'inherit' +}); +proc.on('exit', (code, signal) => { + process.on('exit', () => { + if (grepFile) { + fs.unlinkSync(grepFile); + } + + if (signal) { + process.kill(process.pid, signal); + } else { + process.exit(code); + } + }); +}); + +// terminate children. +process.on('SIGINT', () => { + proc.kill('SIGINT'); // calls runner.abort() + proc.kill('SIGTERM'); // if that didn't work, we're probably in an infinite loop, so make it die. +}); + +function escapeRegExp(pattern) { + return pattern + .replace(/[^-\w\d\s]/g, match => "\\" + match) + .replace(/\s/g, "\\s"); +} \ No newline at end of file diff --git a/src/testRunner/parallel/host.ts b/src/testRunner/parallel/host.ts index 87d193656c3..988e181852a 100644 --- a/src/testRunner/parallel/host.ts +++ b/src/testRunner/parallel/host.ts @@ -16,6 +16,10 @@ namespace Harness.Parallel.Host { const { fork } = require("child_process") as typeof import("child_process"); const { statSync } = require("fs") as typeof import("fs"); + // NOTE: paths for module and types for FailedTestReporter _do not_ line up due to our use of --outFile for run.js + // tslint:disable-next-line:variable-name + const FailedTestReporter = require(path.resolve(__dirname, "../../scripts/failed-tests")) as typeof import("../../../scripts/failed-tests"); + const perfData = readSavedPerfData(configOption); const newTasks: Task[] = []; let tasks: Task[] = []; @@ -54,7 +58,7 @@ namespace Harness.Parallel.Host { interface Worker { process: import("child_process").ChildProcess; accumulatedOutput: string; - currentTasks?: {file: string}[]; + currentTasks?: { file: string }[]; timer?: any; } @@ -115,7 +119,7 @@ namespace Harness.Parallel.Host { update(index: number, percentComplete: number, color: string, title: string | undefined, titleColor?: string) { percentComplete = minMax(percentComplete, 0, 1); - const progressBar = this._progressBars[index] || (this._progressBars[index] = { }); + const progressBar = this._progressBars[index] || (this._progressBars[index] = {}); const width = this._options.width; const n = Math.floor(width * percentComplete); const i = width - n; @@ -177,7 +181,7 @@ namespace Harness.Parallel.Host { return `${perfdataFileNameFragment}${target ? `.${target}` : ""}.json`; } - function readSavedPerfData(target?: string): {[testHash: string]: number} | undefined { + function readSavedPerfData(target?: string): { [testHash: string]: number } | undefined { const perfDataContents = IO.readFile(perfdataFileName(target)); if (perfDataContents) { return JSON.parse(perfDataContents); @@ -189,7 +193,7 @@ namespace Harness.Parallel.Host { return `tsrunner-${runner}://${test}`; } - function startDelayed(perfData: {[testHash: string]: number} | undefined, totalCost: number) { + function startDelayed(perfData: { [testHash: string]: number } | undefined, totalCost: number) { console.log(`Discovered ${tasks.length} unittest suites` + (newTasks.length ? ` and ${newTasks.length} new suites.` : ".")); console.log("Discovering runner-based tests..."); const discoverStart = +(new Date()); @@ -247,7 +251,7 @@ namespace Harness.Parallel.Host { const progressUpdateInterval = 1 / progressBars._options.width; let nextProgress = progressUpdateInterval; - const newPerfData: {[testHash: string]: number} = {}; + const newPerfData: { [testHash: string]: number } = {}; const workers: Worker[] = []; let closedWorkers = 0; @@ -531,10 +535,26 @@ namespace Harness.Parallel.Host { patchStats(consoleReporter.stats); let xunitReporter: import("mocha").reporters.XUnit | undefined; - if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser && process.env.CI === "true") { - xunitReporter = new Mocha.reporters.XUnit(replayRunner, { reporterOptions: { suiteName: "Tests", output: "./TEST-results.xml" } }); - patchStats(xunitReporter.stats); - xunitReporter.write(`\n`); + let failedTestReporter: import("../../../scripts/failed-tests") | undefined; + if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) { + if (process.env.CI === "true") { + xunitReporter = new Mocha.reporters.XUnit(replayRunner, { + reporterOptions: { + suiteName: "Tests", + output: "./TEST-results.xml" + } + }); + patchStats(xunitReporter.stats); + xunitReporter.write(`\n`); + } + else { + failedTestReporter = new FailedTestReporter(replayRunner, { + reporterOptions: { + file: path.resolve(".failed-tests"), + keepFailed + } + }); + } } const savedUseColors = Base.useColors; @@ -551,6 +571,9 @@ namespace Harness.Parallel.Host { if (xunitReporter) { xunitReporter.done(errorResults.length, failures => process.exit(failures)); } + else if (failedTestReporter) { + failedTestReporter.done(errorResults.length, failures => process.exit(failures)); + } else { process.exit(errorResults.length); } diff --git a/src/testRunner/runner.ts b/src/testRunner/runner.ts index d66360458a8..6e430b08734 100644 --- a/src/testRunner/runner.ts +++ b/src/testRunner/runner.ts @@ -62,6 +62,7 @@ let workerCount: number; let runUnitTests: boolean | undefined; let stackTraceLimit: number | "full" | undefined; let noColors = false; +let keepFailed = false; interface TestConfig { light?: boolean; @@ -74,6 +75,7 @@ interface TestConfig { runUnitTests?: boolean; noColors?: boolean; timeout?: number; + keepFailed?: boolean; } interface TaskSet { @@ -102,6 +104,9 @@ function handleTestConfig() { if (testConfig.noColors !== undefined) { noColors = testConfig.noColors; } + if (testConfig.keepFailed) { + keepFailed = true; + } if (testConfig.stackTraceLimit === "full") { (Error).stackTraceLimit = Infinity;