microsoft/TypeScript
1
0
Fork 0
mirror of https://github.com/microsoft/TypeScript.git synced 2026-05-17 21:06:50 -05:00
Code Issues Packages Projects Releases 212 Wiki Activity

Skip costly tests

Browse Source 
1. Add a script to generate a sorted list of most costly tests. A tests'
cost is roughly `runtime% / number of edits`. A slow test that's only
been updated once is much less valuable than a slow test that has
been updated 20 times: the latter test is catching more changes in the
type system.

2. Check in the results of running this script. I want to make the
skipping behaviour deterministic and the same for everybody, even though
you may get slightly better performance by examining only *your* test
changes.

3. Add code to skip tests until it reaches a 5% chance of missing an
edit. Right now this provides a 38% speedup.

Still not done:
4. Make this value configurable.
5. Make the CI configuration specify a 0% chance of missing an edit.
This commit is contained in:
Nathan Shively-Sanders
2019-06-14 13:35:41 -07:00
parent 81f71530c4
commit a852f2feea
4 changed files with 139 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/testRunner/parallel/host.ts
View File

@@ -14,7 +14,9 @@ namespace Harness.Parallel.Host {
const isatty = tty.isatty(1) && tty.isatty(2);
const path = require("path") as typeof import("path");
const { fork } = require("child_process") as typeof import("child_process");
const { statSync } = require("fs") as typeof import("fs");
const { statSync, readFileSync } = require("fs") as typeof import("fs");
const editSkipRate = 0.05
// NOTE: paths for module and types for FailedTestReporter _do not_ line up due to our use of --outFile for run.js
// tslint:disable-next-line:variable-name
@@ -192,7 +194,32 @@ namespace Harness.Parallel.Host {
return `tsrunner-${runner}://${test}`;
}
function startDelayed(perfData: { [testHash: string]: number } | undefined, totalCost: number) {
function skipCostlyTests(tasks: Task[], editSkipRate: number) {
if (statSync('.test-cost.json')) {
const costs = JSON.parse(readFileSync('.test-cost.json', 'utf8')) as {
totalTime: number,
totalEdits: number,
data: Array<{ name: string, time: number, edits: number, costs: number }>
}
let skippedEdits = 0;
let skippedTests = new Set<string>();
let skippedTime = 0;
let i = 0;
for (; i < costs.data.length && (skippedEdits / costs.totalEdits) < editSkipRate; i++) {
skippedEdits += costs.data[i].edits;
skippedTime += costs.data[i].time;
skippedTests.add(costs.data[i].name);
}
console.log(`Skipped ${i} expensive tests; estimated time savings of ${(skippedTime / costs.totalTime * 100).toFixed(2)}% with ${(editSkipRate * 100).toFixed(2)}% chance of missing a test.`)
return tasks.filter(t => !skippedTests.has(t.file));
}
else {
console.log('No cost analysis discovered.');
return tasks;
}
}
function startDelayed(perfData: { [testHash: string]: number } | undefined, totalCost: number, editSkipRate: number) {
console.log(`Discovered ${tasks.length} unittest suites` + (newTasks.length ? ` and ${newTasks.length} new suites.` : "."));
console.log("Discovering runner-based tests...");
const discoverStart = +(new Date());
@@ -231,6 +258,7 @@ namespace Harness.Parallel.Host {
}
tasks.sort((a, b) => a.size - b.size);
tasks = tasks.concat(newTasks);
tasks = skipCostlyTests(tasks, editSkipRate);
const batchCount = workerCount;
const packfraction = 0.9;
const chunkSize = 1000; // ~1KB or 1s for sending batches near the end of a test
@@ -625,6 +653,6 @@ namespace Harness.Parallel.Host {
}
// tslint:disable-next-line:ban
setTimeout(() => startDelayed(perfData, totalCost), 0); // Do real startup on next tick, so all unit tests have been collected
setTimeout(() => startDelayed(perfData, totalCost, editSkipRate), 0); // Do real startup on next tick, so all unit tests have been collected
}
}