Merge pull request #25004 from Microsoft/trackFailedTests

Adds failed test tracking

scripts/build/options.js

@@ -4,7 +4,7 @@ const os = require("os");
 
 /** @type {CommandLineOptions} */
 module.exports = minimist(process.argv.slice(2), {
-    boolean: ["debug", "inspect", "light", "colors", "lint", "soft", "fix"],
+    boolean: ["debug", "inspect", "light", "colors", "lint", "soft", "fix", "failed", "keepFailed"],
     string: ["browser", "tests", "host", "reporter", "stackTraceLimit", "timeout"],
     alias: {
         "b": "browser",
@@ -32,6 +32,8 @@ module.exports = minimist(process.argv.slice(2), {
         lint: process.env.lint || true,
         fix: process.env.fix || process.env.f,
         workers: process.env.workerCount || os.cpus().length,
+        failed: false,
+        keepFailed: false
     }
 });
 
@@ -52,6 +54,8 @@ module.exports = minimist(process.argv.slice(2), {
  * @property {string} reporter
  * @property {string} stackTraceLimit
  * @property {string|number} timeout
+ * @property {boolean} failed
+ * @property {boolean} keepFailed
  * 
  * @typedef {import("minimist").ParsedArgs & TypedOptions} CommandLineOptions
  */

scripts/build/tests.js

@@ -27,14 +27,16 @@ exports.localTest262Baseline = "internal/baselines/test262/local";
  */
 function runConsoleTests(runJs, defaultReporter, runInParallel) {
     let testTimeout = cmdLineOptions.timeout;
+    let tests = cmdLineOptions.tests;
     const lintFlag = cmdLineOptions.lint;
     const debug = cmdLineOptions.debug;
     const inspect = cmdLineOptions.inspect;
-    const tests = cmdLineOptions.tests;
     const runners = cmdLineOptions.runners;
     const light = cmdLineOptions.light;
     const stackTraceLimit = cmdLineOptions.stackTraceLimit;
     const testConfigFile = "test.config";
+    const failed = cmdLineOptions.failed;
+    const keepFailed = cmdLineOptions.keepFailed || failed;
     return cleanTestDirs()
         .then(() => {
             if (fs.existsSync(testConfigFile)) {
@@ -59,8 +61,8 @@ function runConsoleTests(runJs, defaultReporter, runInParallel) {
                 testTimeout = 400000;
             }
 
-            if (tests || runners || light || testTimeout || taskConfigsFolder) {
-                writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout);
+            if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) {
+                writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed);
             }
 
             const colors = cmdLineOptions.colors;
@@ -75,7 +77,8 @@ function runConsoleTests(runJs, defaultReporter, runInParallel) {
             // timeout normally isn"t necessary but Travis-CI has been timing out on compiler baselines occasionally
             // default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
             if (!runInParallel) {
-                args.push("-R", reporter);
+                args.push("-R", "scripts/failed-tests");
+                args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"');
                 if (tests) {
                     args.push("-g", `"${tests}"`);
                 }
@@ -103,7 +106,12 @@ function runConsoleTests(runJs, defaultReporter, runInParallel) {
                 args.push(runJs);
             }
             setNodeEnvToDevelopment();
-            return exec(host, [runJs]);
+            if (failed) {
+                return exec(host, ["scripts/run-failed-tests.js"].concat(args));
+            }
+            else {
+                return exec(host, args);
+            }
         })
         .then(({ exitCode }) => {
             if (exitCode !== 0) return finish(undefined, exitCode);
@@ -148,8 +156,9 @@ exports.cleanTestDirs = cleanTestDirs;
  * @param {string | number} [workerCount]
  * @param {string} [stackTraceLimit]
  * @param {string | number} [timeout]
+ * @param {boolean} [keepFailed]
  */
-function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout) {
+function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed) {
     const testConfigContents = JSON.stringify({
         test: tests ? [tests] : undefined,
         runner: runners ? runners.split(",") : undefined,
@@ -159,6 +168,7 @@ function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCou
         taskConfigsFolder,
         noColor: !cmdLineOptions.colors,
         timeout,
+        keepFailed
     });
     log.info("Running tests with config: " + testConfigContents);
     fs.writeFileSync("test.config", testConfigContents);

scripts/failed-tests.d.ts

@@ -0,0 +1,22 @@
+import Mocha = require("mocha");
+
+export = FailedTestsReporter;
+
+declare class FailedTestsReporter extends Mocha.reporters.Base {
+    passes: Mocha.Test[];
+    failures: Mocha.Test[];
+    reporterOptions: FailedTestsReporter.ReporterOptions;
+    reporter?: Mocha.reporters.Base;
+    constructor(runner: Mocha.Runner, options?: { reporterOptions?: FailedTestsReporter.ReporterOptions });
+    static writeFailures(file: string, passes: ReadonlyArray<Mocha.Test>, failures: ReadonlyArray<Mocha.Test>, keepFailed: boolean, done: (err?: NodeJS.ErrnoException) => void): void;
+    done(failures: number, fn?: (failures: number) => void): void;
+}
+
+declare namespace FailedTestsReporter {
+    interface ReporterOptions {
+        file?: string;
+        keepFailed?: boolean;
+        reporter?: string | Mocha.ReporterConstructor;
+        reporterOptions?: any;
+    }
+}

scripts/failed-tests.js

@@ -0,0 +1,117 @@
+// @ts-check
+const Mocha = require("mocha");
+const path = require("path");
+const fs = require("fs");
+const os = require("os");
+
+/**
+ * .failed-tests reporter
+ * 
+ * @typedef {Object} ReporterOptions
+ * @property {string} [file]
+ * @property {boolean} [keepFailed]
+ * @property {string|Mocha.ReporterConstructor} [reporter]
+ * @property {*} [reporterOptions]
+ */
+class FailedTestsReporter extends Mocha.reporters.Base {
+    /**
+     * @param {Mocha.Runner} runner 
+     * @param {{ reporterOptions?: ReporterOptions }} [options]
+     */
+    constructor(runner, options) {
+        super(runner, options);
+        if (!runner) return;
+
+        const reporterOptions = this.reporterOptions = options.reporterOptions || {};
+        if (reporterOptions.file === undefined) reporterOptions.file = ".failed-tests";
+        if (reporterOptions.keepFailed === undefined) reporterOptions.keepFailed = false;
+        if (reporterOptions.reporter) {
+            /** @type {Mocha.ReporterConstructor} */
+            let reporter;
+            if (typeof reporterOptions.reporter === "function") {
+                reporter = reporterOptions.reporter;
+            }
+            else if (Mocha.reporters[reporterOptions.reporter]) {
+                reporter = Mocha.reporters[reporterOptions.reporter];
+            }
+            else {
+                try {
+                    reporter = require(reporterOptions.reporter);
+                }
+                catch (_) {
+                    reporter = require(path.resolve(process.cwd(), reporterOptions.reporter));
+                }
+            }
+
+            const newOptions = Object.assign({}, options, { reporterOptions: reporterOptions.reporterOptions || {} });
+            this.reporter = new reporter(runner, newOptions);
+        }
+
+        /** @type {Mocha.Test[]} */
+        this.passes = [];
+        
+        /** @type {Mocha.Test[]} */
+        this.failures = [];
+        
+        runner.on("pass", test => this.passes.push(test));
+        runner.on("fail", test => this.failures.push(test));
+    }
+
+    /**
+     * @param {string} file 
+     * @param {ReadonlyArray<Mocha.Test>} passes 
+     * @param {ReadonlyArray<Mocha.Test>} failures 
+     * @param {boolean} keepFailed 
+     * @param {(err?: NodeJS.ErrnoException) => void} done 
+     */
+    static writeFailures(file, passes, failures, keepFailed, done) {
+        const failingTests = new Set(fs.existsSync(file) ? readTests() : undefined);
+        if (failingTests.size > 0) {
+            for (const test of passes) {
+                const title = test.fullTitle().trim();
+                if (title) failingTests.delete(title);
+            }
+        }
+        for (const test of failures) {
+            const title = test.fullTitle().trim();
+            if (title) failingTests.add(title);
+        }
+        if (failingTests.size > 0) {
+            const failed = Array.from(failingTests).join(os.EOL);
+            fs.writeFile(file, failed, "utf8", done);
+        }
+        else if (!keepFailed) {
+            fs.unlink(file, done);
+        }
+        else {
+            done();
+        }
+
+        function readTests() {
+            return fs.readFileSync(file, "utf8")
+                .split(/\r?\n/g)
+                .map(line => line.trim())
+                .filter(line => line.length > 0);
+        }
+    }
+
+    /**
+     * @param {number} failures 
+     * @param {(failures: number) => void} [fn]
+     */
+    done(failures, fn) {
+        FailedTestsReporter.writeFailures(this.reporterOptions.file, this.passes, this.failures, this.reporterOptions.keepFailed || this.stats.tests === 0, (err) => {
+            const reporter = this.reporter;
+            if (reporter && reporter.done) {
+                reporter.done(failures, fn);
+            }
+            else if (fn) {
+                fn(failures);
+            }
+
+            if (err) console.error(err);
+        });
+    }
+}
+
+module.exports = FailedTestsReporter;

scripts/run-failed-tests.js

@@ -0,0 +1,92 @@
+const spawn = require('child_process').spawn;
+const os = require("os");
+const fs = require("fs");
+const path = require("path");
+
+let grep;
+try {
+    const failedTests = fs.readFileSync(".failed-tests", "utf8");
+    grep = failedTests
+        .split(/\r?\n/g)
+        .map(test => test.trim())
+        .filter(test => test.length > 0)
+        .map(escapeRegExp);
+}
+catch (e) {
+    grep = [];
+}
+
+let args = [];
+let waitForGrepValue = false;
+let grepIndex = -1;
+process.argv.slice(2).forEach((arg, index) => {
+    const [flag, value] = arg.split('=');
+    if (flag === "g" || flag === "grep") {
+        grepIndex = index - 1;
+        waitForGrepValue = arg !== flag;
+        if (!waitForGrepValue) grep.push(value.replace(/^"|"$/g, ""));
+        return;
+    }
+    if (waitForGrepValue) {
+        grep.push(arg.replace(/^"|"$/g, ""));
+        waitForGrepValue = false;
+        return;
+    }
+    args.push(arg);
+});
+
+let mocha = "./node_modules/mocha/bin/mocha";
+let grepOption;
+let grepOptionValue;
+let grepFile;
+if (grep.length) {
+    grepOption = "--grep";
+    grepOptionValue = grep.join("|");
+    if (grepOptionValue.length > 20) {
+        grepFile = path.resolve(os.tmpdir(), ".failed-tests.opts");
+        fs.writeFileSync(grepFile, `--grep ${grepOptionValue}`, "utf8");
+        grepOption = "--opts";
+        grepOptionValue = grepFile;
+        mocha = "./node_modules/mocha/bin/_mocha";
+    }
+}
+
+if (grepOption) {
+    if (grepIndex >= 0) {
+        args.splice(grepIndex, 0, grepOption, grepOptionValue);
+    }
+    else {
+        args.push(grepOption, grepOptionValue);
+    }
+}
+
+args.unshift(path.resolve(mocha));
+
+console.log(args.join(" "));
+const proc = spawn(process.execPath, args, {
+    stdio: 'inherit'
+});
+proc.on('exit', (code, signal) => {
+    process.on('exit', () => {
+        if (grepFile) {
+            fs.unlinkSync(grepFile);
+        }
+
+        if (signal) {
+            process.kill(process.pid, signal);
+        } else {
+            process.exit(code);
+        }
+    });
+});
+
+process.on('SIGINT', () => {
+    proc.kill('SIGINT');
+    proc.kill('SIGTERM');
+});
+
+function escapeRegExp(pattern) {
+    return pattern
+        .replace(/[^-\w\d\s]/g, match => "\\" + match)
+        .replace(/\s/g, "\\s");
+}