From 3ef953a8192af2413c0e640fb9f950180b176aa7 Mon Sep 17 00:00:00 2001 From: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com> Date: Fri, 14 Jun 2019 15:59:39 -0700 Subject: [PATCH] Allow passing skipPercent Currently, the default is 5%. 0 gives you 0% time savings 2.5 gives you 29% 5 gives you 38% 10 gives you 50% 20 gives you 65% --- Gulpfile.js | 1 + scripts/build/tests.js | 9 ++++++--- src/testRunner/parallel/host.ts | 14 ++++++-------- src/testRunner/runner.ts | 5 +++++ 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/Gulpfile.js b/Gulpfile.js index bfc4bf43db7..857a2190c17 100644 --- a/Gulpfile.js +++ b/Gulpfile.js @@ -429,6 +429,7 @@ task("runtests-parallel").flags = { " --workers=": "The number of parallel workers to use.", " --timeout=": "Overrides the default test timeout.", " --built": "Compile using the built version of the compiler.", + " --skipPercent=": "Skip expensive tests with chance to miss an edit.", }; task("diff", () => exec(getDiffTool(), [refBaseline, localBaseline], { ignoreExitCode: true })); diff --git a/scripts/build/tests.js b/scripts/build/tests.js index 0ac65002ab8..2843659f511 100644 --- a/scripts/build/tests.js +++ b/scripts/build/tests.js @@ -31,6 +31,7 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode, const inspect = cmdLineOptions.inspect; const runners = cmdLineOptions.runners; const light = cmdLineOptions.light; + const skipPercent = cmdLineOptions.skipPercent; const stackTraceLimit = cmdLineOptions.stackTraceLimit; const testConfigFile = "test.config"; const failed = cmdLineOptions.failed; @@ -62,8 +63,8 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode, testTimeout = 400000; } - if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) { - writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed); + if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed || skipPercent) { + writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed); } const colors = cmdLineOptions.colors; @@ -158,17 +159,19 @@ exports.cleanTestDirs = cleanTestDirs; * @param {string} tests * @param {string} runners * @param {boolean} light + * @param {string} skipPercent * @param {string} [taskConfigsFolder] * @param {string | number} [workerCount] * @param {string} [stackTraceLimit] * @param {string | number} [timeout] * @param {boolean} [keepFailed] */ -function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed) { +function writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed) { const testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, runners: runners ? runners.split(",") : undefined, light, + skipPercent, workerCount, stackTraceLimit, taskConfigsFolder, diff --git a/src/testRunner/parallel/host.ts b/src/testRunner/parallel/host.ts index 0cb53eedd98..74a54fad5dd 100644 --- a/src/testRunner/parallel/host.ts +++ b/src/testRunner/parallel/host.ts @@ -16,8 +16,6 @@ namespace Harness.Parallel.Host { const { fork } = require("child_process") as typeof import("child_process"); const { statSync, readFileSync } = require("fs") as typeof import("fs"); - const editSkipRate = 0.05 - // NOTE: paths for module and types for FailedTestReporter _do not_ line up due to our use of --outFile for run.js // tslint:disable-next-line:variable-name const FailedTestReporter = require(path.resolve(__dirname, "../../scripts/failed-tests")) as typeof import("../../../scripts/failed-tests"); @@ -194,7 +192,7 @@ namespace Harness.Parallel.Host { return `tsrunner-${runner}://${test}`; } - function skipCostlyTests(tasks: Task[], editSkipRate: number) { + function skipCostlyTests(tasks: Task[]) { if (statSync('.test-cost.json')) { const costs = JSON.parse(readFileSync('.test-cost.json', 'utf8')) as { totalTime: number, @@ -205,12 +203,12 @@ namespace Harness.Parallel.Host { let skippedTests = new Set(); let skippedTime = 0; let i = 0; - for (; i < costs.data.length && (skippedEdits / costs.totalEdits) < editSkipRate; i++) { + for (; i < costs.data.length && (skippedEdits / costs.totalEdits) < (skipPercent / 100); i++) { skippedEdits += costs.data[i].edits; skippedTime += costs.data[i].time; skippedTests.add(costs.data[i].name); } - console.log(`Skipped ${i} expensive tests; estimated time savings of ${(skippedTime / costs.totalTime * 100).toFixed(2)}% with ${(editSkipRate * 100).toFixed(2)}% chance of missing a test.`) + console.log(`Skipped ${i} expensive tests; estimated time savings of ${(skippedTime / costs.totalTime * 100).toFixed(2)}% with ${skipPercent.toFixed(2)}% chance of missing a test.`) return tasks.filter(t => !skippedTests.has(t.file)); } else { @@ -219,7 +217,7 @@ namespace Harness.Parallel.Host { } } - function startDelayed(perfData: { [testHash: string]: number } | undefined, totalCost: number, editSkipRate: number) { + function startDelayed(perfData: { [testHash: string]: number } | undefined, totalCost: number) { console.log(`Discovered ${tasks.length} unittest suites` + (newTasks.length ? ` and ${newTasks.length} new suites.` : ".")); console.log("Discovering runner-based tests..."); const discoverStart = +(new Date()); @@ -258,7 +256,7 @@ namespace Harness.Parallel.Host { } tasks.sort((a, b) => a.size - b.size); tasks = tasks.concat(newTasks); - tasks = skipCostlyTests(tasks, editSkipRate); + tasks = skipCostlyTests(tasks); const batchCount = workerCount; const packfraction = 0.9; const chunkSize = 1000; // ~1KB or 1s for sending batches near the end of a test @@ -653,6 +651,6 @@ namespace Harness.Parallel.Host { } // tslint:disable-next-line:ban - setTimeout(() => startDelayed(perfData, totalCost, editSkipRate), 0); // Do real startup on next tick, so all unit tests have been collected + setTimeout(() => startDelayed(perfData, totalCost), 0); // Do real startup on next tick, so all unit tests have been collected } } diff --git a/src/testRunner/runner.ts b/src/testRunner/runner.ts index be46d935297..ecb925cbe74 100644 --- a/src/testRunner/runner.ts +++ b/src/testRunner/runner.ts @@ -63,6 +63,7 @@ let runUnitTests: boolean | undefined; let stackTraceLimit: number | "full" | undefined; let noColors = false; let keepFailed = false; +let skipPercent = 5; interface TestConfig { light?: boolean; @@ -76,6 +77,7 @@ interface TestConfig { noColors?: boolean; timeout?: number; keepFailed?: boolean; + skipPercent?: number; } interface TaskSet { @@ -107,6 +109,9 @@ function handleTestConfig() { if (testConfig.keepFailed) { keepFailed = true; } + if (testConfig.skipPercent !== undefined) { + skipPercent = testConfig.skipPercent; + } if (testConfig.stackTraceLimit === "full") { (Error).stackTraceLimit = Infinity;