diff --git a/.gitignore b/.gitignore index b5aecdbe35d..fdcea6a6800 100644 --- a/.gitignore +++ b/.gitignore @@ -73,4 +73,5 @@ tests/cases/user/*/**/*.d.ts !tests/cases/user/zone.js/ !tests/cases/user/bignumber.js/ !tests/cases/user/discord.js/ -tests/baselines/reference/dt \ No newline at end of file +tests/baselines/reference/dt +.failed-tests \ No newline at end of file diff --git a/Jakefile.js b/Jakefile.js index ce5576036f4..6e99b7fb0ae 100644 --- a/Jakefile.js +++ b/Jakefile.js @@ -410,6 +410,8 @@ function runConsoleTests(defaultReporter, runInParallel) { const runners = process.env.runners || process.env.runner || process.env.ru; const tests = process.env.test || process.env.tests || process.env.t; const light = process.env.light === undefined || process.env.light !== "false"; + const failed = process.env.failed; + const keepFailed = process.env.keepFailed || failed; const stackTraceLimit = process.env.stackTraceLimit; const colorsFlag = process.env.color || process.env.colors; const colors = colorsFlag !== "false" && colorsFlag !== "0"; @@ -440,8 +442,8 @@ function runConsoleTests(defaultReporter, runInParallel) { testTimeout = 800000; } - if (tests || runners || light || testTimeout || taskConfigsFolder) { - writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout); + if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) { + writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout, keepFailed); } // timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally @@ -449,7 +451,8 @@ function runConsoleTests(defaultReporter, runInParallel) { if (!runInParallel) { var startTime = Travis.mark(); var args = []; - args.push("-R", reporter); + args.push("-R", "scripts/failed-tests"); + args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"'); if (tests) args.push("-g", `"${tests}"`); args.push(colors ? "--colors" : "--no-colors"); if (bail) args.push("--bail"); @@ -460,7 +463,14 @@ function runConsoleTests(defaultReporter, runInParallel) { } args.push(Paths.builtLocalRun); - var cmd = "mocha " + args.join(" "); + var cmd; + if (failed) { + args.unshift("scripts/run-failed-tests.js"); + cmd = host + " " + args.join(" "); + } + else { + cmd = "mocha " + args.join(" "); + } var savedNodeEnv = process.env.NODE_ENV; process.env.NODE_ENV = "development"; exec(cmd, function () { @@ -521,7 +531,7 @@ function runConsoleTests(defaultReporter, runInParallel) { } // used to pass data from jake command line directly to run.js -function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout) { +function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout, keepFailed) { var testConfigContents = JSON.stringify({ runners: runners ? runners.split(",") : undefined, test: tests ? [tests] : undefined, @@ -530,7 +540,8 @@ function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCou taskConfigsFolder: taskConfigsFolder, stackTraceLimit: stackTraceLimit, noColor: !colors, - timeout: testTimeout + timeout: testTimeout, + keepFailed: keepFailed }); fs.writeFileSync('test.config', testConfigContents, { encoding: "utf-8" }); } diff --git a/scripts/build/options.js b/scripts/build/options.js index df0fa223026..b69358f0f98 100644 --- a/scripts/build/options.js +++ b/scripts/build/options.js @@ -4,7 +4,7 @@ const os = require("os"); /** @type {CommandLineOptions} */ module.exports = minimist(process.argv.slice(2), { - boolean: ["debug", "inspect", "light", "colors", "lint", "soft", "fix"], + boolean: ["debug", "inspect", "light", "colors", "lint", "soft", "fix", "failed", "keepFailed"], string: ["browser", "tests", "host", "reporter", "stackTraceLimit", "timeout"], alias: { "b": "browser", @@ -32,6 +32,8 @@ module.exports = minimist(process.argv.slice(2), { lint: process.env.lint || true, fix: process.env.fix || process.env.f, workers: process.env.workerCount || os.cpus().length, + failed: false, + keepFailed: false } }); @@ -52,6 +54,8 @@ module.exports = minimist(process.argv.slice(2), { * @property {string} reporter * @property {string} stackTraceLimit * @property {string|number} timeout + * @property {boolean} failed + * @property {boolean} keepFailed * * @typedef {import("minimist").ParsedArgs & TypedOptions} CommandLineOptions */ diff --git a/scripts/build/tests.js b/scripts/build/tests.js index 60b72e64cdd..36615149d70 100644 --- a/scripts/build/tests.js +++ b/scripts/build/tests.js @@ -27,14 +27,16 @@ exports.localTest262Baseline = "internal/baselines/test262/local"; */ function runConsoleTests(runJs, defaultReporter, runInParallel) { let testTimeout = cmdLineOptions.timeout; + let tests = cmdLineOptions.tests; const lintFlag = cmdLineOptions.lint; const debug = cmdLineOptions.debug; const inspect = cmdLineOptions.inspect; - const tests = cmdLineOptions.tests; const runners = cmdLineOptions.runners; const light = cmdLineOptions.light; const stackTraceLimit = cmdLineOptions.stackTraceLimit; const testConfigFile = "test.config"; + const failed = cmdLineOptions.failed; + const keepFailed = cmdLineOptions.keepFailed || failed; return cleanTestDirs() .then(() => { if (fs.existsSync(testConfigFile)) { @@ -59,8 +61,8 @@ function runConsoleTests(runJs, defaultReporter, runInParallel) { testTimeout = 400000; } - if (tests || runners || light || testTimeout || taskConfigsFolder) { - writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout); + if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) { + writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed); } const colors = cmdLineOptions.colors; @@ -75,7 +77,8 @@ function runConsoleTests(runJs, defaultReporter, runInParallel) { // timeout normally isn"t necessary but Travis-CI has been timing out on compiler baselines occasionally // default timeout is 2sec which really should be enough, but maybe we just need a small amount longer if (!runInParallel) { - args.push("-R", reporter); + args.push("-R", "scripts/failed-tests"); + args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"'); if (tests) { args.push("-g", `"${tests}"`); } @@ -103,7 +106,12 @@ function runConsoleTests(runJs, defaultReporter, runInParallel) { args.push(runJs); } setNodeEnvToDevelopment(); - return exec(host, [runJs]); + if (failed) { + return exec(host, ["scripts/run-failed-tests.js"].concat(args)); + } + else { + return exec(host, args); + } }) .then(({ exitCode }) => { if (exitCode !== 0) return finish(undefined, exitCode); @@ -148,8 +156,9 @@ exports.cleanTestDirs = cleanTestDirs; * @param {string | number} [workerCount] * @param {string} [stackTraceLimit] * @param {string | number} [timeout] + * @param {boolean} [keepFailed] */ -function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout) { +function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed) { const testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, runner: runners ? runners.split(",") : undefined, @@ -159,6 +168,7 @@ function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCou taskConfigsFolder, noColor: !cmdLineOptions.colors, timeout, + keepFailed }); log.info("Running tests with config: " + testConfigContents); fs.writeFileSync("test.config", testConfigContents); diff --git a/scripts/failed-tests.d.ts b/scripts/failed-tests.d.ts new file mode 100644 index 00000000000..e82c0b178e8 --- /dev/null +++ b/scripts/failed-tests.d.ts @@ -0,0 +1,22 @@ +import Mocha = require("mocha"); + +export = FailedTestsReporter; + +declare class FailedTestsReporter extends Mocha.reporters.Base { + passes: Mocha.Test[]; + failures: Mocha.Test[]; + reporterOptions: FailedTestsReporter.ReporterOptions; + reporter?: Mocha.reporters.Base; + constructor(runner: Mocha.Runner, options?: { reporterOptions?: FailedTestsReporter.ReporterOptions }); + static writeFailures(file: string, passes: ReadonlyArray, failures: ReadonlyArray, keepFailed: boolean, done: (err?: NodeJS.ErrnoException) => void): void; + done(failures: number, fn?: (failures: number) => void): void; +} + +declare namespace FailedTestsReporter { + interface ReporterOptions { + file?: string; + keepFailed?: boolean; + reporter?: string | Mocha.ReporterConstructor; + reporterOptions?: any; + } +} \ No newline at end of file diff --git a/scripts/failed-tests.js b/scripts/failed-tests.js new file mode 100644 index 00000000000..ca26c432948 --- /dev/null +++ b/scripts/failed-tests.js @@ -0,0 +1,117 @@ +// @ts-check +const Mocha = require("mocha"); +const path = require("path"); +const fs = require("fs"); +const os = require("os"); + +/** + * .failed-tests reporter + * + * @typedef {Object} ReporterOptions + * @property {string} [file] + * @property {boolean} [keepFailed] + * @property {string|Mocha.ReporterConstructor} [reporter] + * @property {*} [reporterOptions] + */ +class FailedTestsReporter extends Mocha.reporters.Base { + /** + * @param {Mocha.Runner} runner + * @param {{ reporterOptions?: ReporterOptions }} [options] + */ + constructor(runner, options) { + super(runner, options); + if (!runner) return; + + const reporterOptions = this.reporterOptions = options.reporterOptions || {}; + if (reporterOptions.file === undefined) reporterOptions.file = ".failed-tests"; + if (reporterOptions.keepFailed === undefined) reporterOptions.keepFailed = false; + if (reporterOptions.reporter) { + /** @type {Mocha.ReporterConstructor} */ + let reporter; + if (typeof reporterOptions.reporter === "function") { + reporter = reporterOptions.reporter; + } + else if (Mocha.reporters[reporterOptions.reporter]) { + reporter = Mocha.reporters[reporterOptions.reporter]; + } + else { + try { + reporter = require(reporterOptions.reporter); + } + catch (_) { + reporter = require(path.resolve(process.cwd(), reporterOptions.reporter)); + } + } + + const newOptions = Object.assign({}, options, { reporterOptions: reporterOptions.reporterOptions || {} }); + this.reporter = new reporter(runner, newOptions); + } + + /** @type {Mocha.Test[]} */ + this.passes = []; + + /** @type {Mocha.Test[]} */ + this.failures = []; + + runner.on("pass", test => this.passes.push(test)); + runner.on("fail", test => this.failures.push(test)); + } + + /** + * @param {string} file + * @param {ReadonlyArray} passes + * @param {ReadonlyArray} failures + * @param {boolean} keepFailed + * @param {(err?: NodeJS.ErrnoException) => void} done + */ + static writeFailures(file, passes, failures, keepFailed, done) { + const failingTests = new Set(fs.existsSync(file) ? readTests() : undefined); + if (failingTests.size > 0) { + for (const test of passes) { + const title = test.fullTitle().trim(); + if (title) failingTests.delete(title); + } + } + for (const test of failures) { + const title = test.fullTitle().trim(); + if (title) failingTests.add(title); + } + if (failingTests.size > 0) { + const failed = Array.from(failingTests).join(os.EOL); + fs.writeFile(file, failed, "utf8", done); + } + else if (!keepFailed) { + fs.unlink(file, done); + } + else { + done(); + } + + function readTests() { + return fs.readFileSync(file, "utf8") + .split(/\r?\n/g) + .map(line => line.trim()) + .filter(line => line.length > 0); + } + } + + /** + * @param {number} failures + * @param {(failures: number) => void} [fn] + */ + done(failures, fn) { + FailedTestsReporter.writeFailures(this.reporterOptions.file, this.passes, this.failures, this.reporterOptions.keepFailed || this.stats.tests === 0, (err) => { + const reporter = this.reporter; + if (reporter && reporter.done) { + reporter.done(failures, fn); + } + else if (fn) { + fn(failures); + } + + if (err) console.error(err); + }); + } +} + +module.exports = FailedTestsReporter; \ No newline at end of file diff --git a/scripts/run-failed-tests.js b/scripts/run-failed-tests.js new file mode 100644 index 00000000000..876c4a34eec --- /dev/null +++ b/scripts/run-failed-tests.js @@ -0,0 +1,92 @@ +const spawn = require('child_process').spawn; +const os = require("os"); +const fs = require("fs"); +const path = require("path"); + +let grep; +try { + const failedTests = fs.readFileSync(".failed-tests", "utf8"); + grep = failedTests + .split(/\r?\n/g) + .map(test => test.trim()) + .filter(test => test.length > 0) + .map(escapeRegExp); +} +catch (e) { + grep = []; +} + +let args = []; +let waitForGrepValue = false; +let grepIndex = -1; +process.argv.slice(2).forEach((arg, index) => { + const [flag, value] = arg.split('='); + if (flag === "g" || flag === "grep") { + grepIndex = index - 1; + waitForGrepValue = arg !== flag; + if (!waitForGrepValue) grep.push(value.replace(/^"|"$/g, "")); + return; + } + if (waitForGrepValue) { + grep.push(arg.replace(/^"|"$/g, "")); + waitForGrepValue = false; + return; + } + args.push(arg); +}); + +let mocha = "./node_modules/mocha/bin/mocha"; +let grepOption; +let grepOptionValue; +let grepFile; +if (grep.length) { + grepOption = "--grep"; + grepOptionValue = grep.join("|"); + if (grepOptionValue.length > 20) { + grepFile = path.resolve(os.tmpdir(), ".failed-tests.opts"); + fs.writeFileSync(grepFile, `--grep ${grepOptionValue}`, "utf8"); + grepOption = "--opts"; + grepOptionValue = grepFile; + mocha = "./node_modules/mocha/bin/_mocha"; + } +} + +if (grepOption) { + if (grepIndex >= 0) { + args.splice(grepIndex, 0, grepOption, grepOptionValue); + } + else { + args.push(grepOption, grepOptionValue); + } +} + +args.unshift(path.resolve(mocha)); + +console.log(args.join(" ")); +const proc = spawn(process.execPath, args, { + stdio: 'inherit' +}); +proc.on('exit', (code, signal) => { + process.on('exit', () => { + if (grepFile) { + fs.unlinkSync(grepFile); + } + + if (signal) { + process.kill(process.pid, signal); + } else { + process.exit(code); + } + }); +}); + +process.on('SIGINT', () => { + proc.kill('SIGINT'); + proc.kill('SIGTERM'); +}); + +function escapeRegExp(pattern) { + return pattern + .replace(/[^-\w\d\s]/g, match => "\\" + match) + .replace(/\s/g, "\\s"); +} \ No newline at end of file diff --git a/src/testRunner/parallel/host.ts b/src/testRunner/parallel/host.ts index 87d193656c3..988e181852a 100644 --- a/src/testRunner/parallel/host.ts +++ b/src/testRunner/parallel/host.ts @@ -16,6 +16,10 @@ namespace Harness.Parallel.Host { const { fork } = require("child_process") as typeof import("child_process"); const { statSync } = require("fs") as typeof import("fs"); + // NOTE: paths for module and types for FailedTestReporter _do not_ line up due to our use of --outFile for run.js + // tslint:disable-next-line:variable-name + const FailedTestReporter = require(path.resolve(__dirname, "../../scripts/failed-tests")) as typeof import("../../../scripts/failed-tests"); + const perfData = readSavedPerfData(configOption); const newTasks: Task[] = []; let tasks: Task[] = []; @@ -54,7 +58,7 @@ namespace Harness.Parallel.Host { interface Worker { process: import("child_process").ChildProcess; accumulatedOutput: string; - currentTasks?: {file: string}[]; + currentTasks?: { file: string }[]; timer?: any; } @@ -115,7 +119,7 @@ namespace Harness.Parallel.Host { update(index: number, percentComplete: number, color: string, title: string | undefined, titleColor?: string) { percentComplete = minMax(percentComplete, 0, 1); - const progressBar = this._progressBars[index] || (this._progressBars[index] = { }); + const progressBar = this._progressBars[index] || (this._progressBars[index] = {}); const width = this._options.width; const n = Math.floor(width * percentComplete); const i = width - n; @@ -177,7 +181,7 @@ namespace Harness.Parallel.Host { return `${perfdataFileNameFragment}${target ? `.${target}` : ""}.json`; } - function readSavedPerfData(target?: string): {[testHash: string]: number} | undefined { + function readSavedPerfData(target?: string): { [testHash: string]: number } | undefined { const perfDataContents = IO.readFile(perfdataFileName(target)); if (perfDataContents) { return JSON.parse(perfDataContents); @@ -189,7 +193,7 @@ namespace Harness.Parallel.Host { return `tsrunner-${runner}://${test}`; } - function startDelayed(perfData: {[testHash: string]: number} | undefined, totalCost: number) { + function startDelayed(perfData: { [testHash: string]: number } | undefined, totalCost: number) { console.log(`Discovered ${tasks.length} unittest suites` + (newTasks.length ? ` and ${newTasks.length} new suites.` : ".")); console.log("Discovering runner-based tests..."); const discoverStart = +(new Date()); @@ -247,7 +251,7 @@ namespace Harness.Parallel.Host { const progressUpdateInterval = 1 / progressBars._options.width; let nextProgress = progressUpdateInterval; - const newPerfData: {[testHash: string]: number} = {}; + const newPerfData: { [testHash: string]: number } = {}; const workers: Worker[] = []; let closedWorkers = 0; @@ -531,10 +535,26 @@ namespace Harness.Parallel.Host { patchStats(consoleReporter.stats); let xunitReporter: import("mocha").reporters.XUnit | undefined; - if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser && process.env.CI === "true") { - xunitReporter = new Mocha.reporters.XUnit(replayRunner, { reporterOptions: { suiteName: "Tests", output: "./TEST-results.xml" } }); - patchStats(xunitReporter.stats); - xunitReporter.write(`\n`); + let failedTestReporter: import("../../../scripts/failed-tests") | undefined; + if (Utils.getExecutionEnvironment() !== Utils.ExecutionEnvironment.Browser) { + if (process.env.CI === "true") { + xunitReporter = new Mocha.reporters.XUnit(replayRunner, { + reporterOptions: { + suiteName: "Tests", + output: "./TEST-results.xml" + } + }); + patchStats(xunitReporter.stats); + xunitReporter.write(`\n`); + } + else { + failedTestReporter = new FailedTestReporter(replayRunner, { + reporterOptions: { + file: path.resolve(".failed-tests"), + keepFailed + } + }); + } } const savedUseColors = Base.useColors; @@ -551,6 +571,9 @@ namespace Harness.Parallel.Host { if (xunitReporter) { xunitReporter.done(errorResults.length, failures => process.exit(failures)); } + else if (failedTestReporter) { + failedTestReporter.done(errorResults.length, failures => process.exit(failures)); + } else { process.exit(errorResults.length); } diff --git a/src/testRunner/runner.ts b/src/testRunner/runner.ts index d66360458a8..6e430b08734 100644 --- a/src/testRunner/runner.ts +++ b/src/testRunner/runner.ts @@ -62,6 +62,7 @@ let workerCount: number; let runUnitTests: boolean | undefined; let stackTraceLimit: number | "full" | undefined; let noColors = false; +let keepFailed = false; interface TestConfig { light?: boolean; @@ -74,6 +75,7 @@ interface TestConfig { runUnitTests?: boolean; noColors?: boolean; timeout?: number; + keepFailed?: boolean; } interface TaskSet { @@ -102,6 +104,9 @@ function handleTestConfig() { if (testConfig.noColors !== undefined) { noColors = testConfig.noColors; } + if (testConfig.keepFailed) { + keepFailed = true; + } if (testConfig.stackTraceLimit === "full") { (Error).stackTraceLimit = Infinity;