diff --git a/Gulpfile.js b/Gulpfile.js index a8f5748271d..42987e4fd8d 100644 --- a/Gulpfile.js +++ b/Gulpfile.js @@ -415,6 +415,8 @@ task("runtests").flags = { " --no-lint": "Disables lint", " --timeout=": "Overrides the default test timeout.", " --built": "Compile using the built version of the compiler.", + " --shards": "Total number of shards running tests (default: 1)", + " --shardId": "1-based ID of this shard (default: 1)", } const runTestsParallel = () => runConsoleTests("built/local/run.js", "min", /*runInParallel*/ true, /*watchMode*/ false); @@ -430,6 +432,8 @@ task("runtests-parallel").flags = { " --timeout=": "Overrides the default test timeout.", " --built": "Compile using the built version of the compiler.", " --skipPercent=": "Skip expensive tests with chance to miss an edit. Default 5%.", + " --shards": "Total number of shards running tests (default: 1)", + " --shardId": "1-based ID of this shard (default: 1)", }; task("diff", () => exec(getDiffTool(), [refBaseline, localBaseline], { ignoreExitCode: true })); diff --git a/scripts/build/options.js b/scripts/build/options.js index c93a265ff88..fecac01dd05 100644 --- a/scripts/build/options.js +++ b/scripts/build/options.js @@ -5,7 +5,7 @@ const os = require("os"); /** @type {CommandLineOptions} */ module.exports = minimist(process.argv.slice(2), { boolean: ["debug", "dirty", "inspect", "light", "colors", "lint", "lkg", "soft", "fix", "failed", "keepFailed", "force", "built"], - string: ["browser", "tests", "host", "reporter", "stackTraceLimit", "timeout"], + string: ["browser", "tests", "host", "reporter", "stackTraceLimit", "timeout", "shards", "shardId"], alias: { "b": "browser", "d": "debug", "debug-brk": "debug", diff --git a/scripts/build/tests.js b/scripts/build/tests.js index a15eb17093d..20d2a4c7c2a 100644 --- a/scripts/build/tests.js +++ b/scripts/build/tests.js @@ -36,6 +36,8 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode, const testConfigFile = "test.config"; const failed = cmdLineOptions.failed; const keepFailed = cmdLineOptions.keepFailed; + const shards = +cmdLineOptions.shards || undefined; + const shardId = +cmdLineOptions.shardId || undefined; if (!cmdLineOptions.dirty) { await cleanTestDirs(); cancelToken.throwIfCancellationRequested(); @@ -63,8 +65,8 @@ async function runConsoleTests(runJs, defaultReporter, runInParallel, watchMode, testTimeout = 400000; } - if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed || skipPercent !== undefined) { - writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed); + if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed || skipPercent !== undefined || shards || shardId) { + writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed, shards, shardId); } const colors = cmdLineOptions.colors; @@ -165,8 +167,10 @@ exports.cleanTestDirs = cleanTestDirs; * @param {string} [stackTraceLimit] * @param {string | number} [timeout] * @param {boolean} [keepFailed] + * @param {number | undefined} [shards] + * @param {number | undefined} [shardId] */ -function writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed) { +function writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed, shards, shardId) { const testConfigContents = JSON.stringify({ test: tests ? [tests] : undefined, runners: runners ? runners.split(",") : undefined, @@ -177,7 +181,9 @@ function writeTestConfigFile(tests, runners, light, skipPercent, taskConfigsFold taskConfigsFolder, noColor: !cmdLineOptions.colors, timeout, - keepFailed + keepFailed, + shards, + shardId }); log.info("Running tests with config: " + testConfigContents); fs.writeFileSync("test.config", testConfigContents); diff --git a/src/harness/harness.ts b/src/harness/harness.ts index c6842115095..fe5ec344322 100644 --- a/src/harness/harness.ts +++ b/src/harness/harness.ts @@ -501,7 +501,7 @@ namespace Harness { } function enumerateTestFiles(runner: RunnerBase) { - return runner.enumerateTestFiles(); + return runner.getTestFiles(); } function listFiles(path: string, spec: RegExp, options: { recursive?: boolean } = {}) { diff --git a/src/harness/runnerbase.ts b/src/harness/runnerbase.ts index ce56772927e..70045c54c80 100644 --- a/src/harness/runnerbase.ts +++ b/src/harness/runnerbase.ts @@ -2,6 +2,9 @@ type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | type CompilerTestKind = "conformance" | "compiler"; type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server"; +let shards = 1; +let shardId = 1; + abstract class RunnerBase { // contains the tests to run public tests: (string | Harness.FileBasedTest)[] = []; @@ -19,6 +22,14 @@ abstract class RunnerBase { abstract enumerateTestFiles(): (string | Harness.FileBasedTest)[]; + getTestFiles(): ReturnType { + const all = this.enumerateTestFiles(); + if (shards === 1) { + return all as ReturnType; + } + return all.filter((_val, idx) => idx % shards === (shardId - 1)) as ReturnType; + } + /** The working directory where tests are found. Needed for batch testing where the input path will differ from the output path inside baselines */ public workingDirectory = ""; diff --git a/src/testRunner/externalCompileRunner.ts b/src/testRunner/externalCompileRunner.ts index 4be0d1a67ab..dbe7bc5854f 100644 --- a/src/testRunner/externalCompileRunner.ts +++ b/src/testRunner/externalCompileRunner.ts @@ -24,7 +24,7 @@ abstract class ExternalCompileRunnerBase extends RunnerBase { */ initializeTests(): void { // Read in and evaluate the test list - const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles(); + const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles(); // tslint:disable-next-line:no-this-assignment const cls = this; @@ -113,7 +113,7 @@ class DockerfileRunner extends ExternalCompileRunnerBase { } initializeTests(): void { // Read in and evaluate the test list - const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles(); + const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles(); // tslint:disable-next-line:no-this-assignment const cls = this; diff --git a/src/testRunner/parallel/host.ts b/src/testRunner/parallel/host.ts index 7780cfbd646..ee66fd738b0 100644 --- a/src/testRunner/parallel/host.ts +++ b/src/testRunner/parallel/host.ts @@ -222,7 +222,7 @@ namespace Harness.Parallel.Host { console.log("Discovering runner-based tests..."); const discoverStart = +(new Date()); for (const runner of runners) { - for (const test of runner.enumerateTestFiles()) { + for (const test of runner.getTestFiles()) { const file = typeof test === "string" ? test : test.file; let size: number; if (!perfData) { diff --git a/src/testRunner/projectsRunner.ts b/src/testRunner/projectsRunner.ts index f4e51afab4c..ea8efd838d4 100644 --- a/src/testRunner/projectsRunner.ts +++ b/src/testRunner/projectsRunner.ts @@ -32,7 +32,11 @@ namespace project { export class ProjectRunner extends RunnerBase { public enumerateTestFiles() { - return this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true }); + const all = this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true }); + if (shards === 1) { + return all; + } + return all.filter((_val, idx) => idx % shards === (shardId - 1)); } public kind(): TestRunnerKind { diff --git a/src/testRunner/runner.ts b/src/testRunner/runner.ts index 685937196cb..3532f972230 100644 --- a/src/testRunner/runner.ts +++ b/src/testRunner/runner.ts @@ -80,6 +80,8 @@ interface TestConfig { timeout?: number; keepFailed?: boolean; skipPercent?: number; + shardId?: number; + shards?: number; } interface TaskSet { @@ -114,6 +116,12 @@ function handleTestConfig() { if (testConfig.skipPercent !== undefined) { skipPercent = testConfig.skipPercent; } + if (testConfig.shardId) { + shardId = testConfig.shardId; + } + if (testConfig.shards) { + shards = testConfig.shards; + } if (testConfig.stackTraceLimit === "full") { (Error).stackTraceLimit = Infinity; @@ -129,6 +137,9 @@ function handleTestConfig() { const runnerConfig = testConfig.runners || testConfig.test; if (runnerConfig && runnerConfig.length > 0) { + if (testConfig.runners) { + runUnitTests = runnerConfig.indexOf("unittest") !== -1; + } for (const option of runnerConfig) { if (!option) { continue; diff --git a/src/testRunner/rwcRunner.ts b/src/testRunner/rwcRunner.ts index 729ed167fff..b510b98668a 100644 --- a/src/testRunner/rwcRunner.ts +++ b/src/testRunner/rwcRunner.ts @@ -225,7 +225,7 @@ class RWCRunner extends RunnerBase { */ public initializeTests(): void { // Read in and evaluate the test list - for (const test of this.tests && this.tests.length ? this.tests : this.enumerateTestFiles()) { + for (const test of this.tests && this.tests.length ? this.tests : this.getTestFiles()) { this.runTest(typeof test === "string" ? test : test.file); } } diff --git a/src/testRunner/test262Runner.ts b/src/testRunner/test262Runner.ts index 3edb065b325..c435c3e23ca 100644 --- a/src/testRunner/test262Runner.ts +++ b/src/testRunner/test262Runner.ts @@ -97,7 +97,7 @@ class Test262BaselineRunner extends RunnerBase { public initializeTests() { // this will set up a series of describe/it blocks to run between the setup and cleanup phases if (this.tests.length === 0) { - const testFiles = this.enumerateTestFiles(); + const testFiles = this.getTestFiles(); testFiles.forEach(fn => { this.runTest(fn); });