Add support for sharding tests across multiple workers (#32173)

* Add support for sharding tests across multiple workers

* Disable unittests when runners are expressly provided (unless they contain the unittest runner)

src/harness/harness.ts

@@ -501,7 +501,7 @@ namespace Harness {
         }
 
         function enumerateTestFiles(runner: RunnerBase) {
-            return runner.enumerateTestFiles();
+            return runner.getTestFiles();
         }
 
         function listFiles(path: string, spec: RegExp, options: { recursive?: boolean } = {}) {

src/harness/runnerbase.ts

@@ -2,6 +2,9 @@ type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" |
 type CompilerTestKind = "conformance" | "compiler";
 type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server";
 
+let shards = 1;
+let shardId = 1;
+
 abstract class RunnerBase {
     // contains the tests to run
     public tests: (string | Harness.FileBasedTest)[] = [];
@@ -19,6 +22,14 @@ abstract class RunnerBase {
 
     abstract enumerateTestFiles(): (string | Harness.FileBasedTest)[];
 
+    getTestFiles(): ReturnType<this["enumerateTestFiles"]> {
+        const all = this.enumerateTestFiles();
+        if (shards === 1) {
+            return all as ReturnType<this["enumerateTestFiles"]>;
+        }
+        return all.filter((_val, idx) => idx % shards === (shardId - 1)) as ReturnType<this["enumerateTestFiles"]>;
+    }
+
     /** The working directory where tests are found. Needed for batch testing where the input path will differ from the output path inside baselines */
     public workingDirectory = "";
 

src/testRunner/externalCompileRunner.ts

@@ -24,7 +24,7 @@ abstract class ExternalCompileRunnerBase extends RunnerBase {
      */
     initializeTests(): void {
         // Read in and evaluate the test list
-        const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles();
+        const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles();
 
         // tslint:disable-next-line:no-this-assignment
         const cls = this;
@@ -113,7 +113,7 @@ class DockerfileRunner extends ExternalCompileRunnerBase {
     }
     initializeTests(): void {
         // Read in and evaluate the test list
-        const testList = this.tests && this.tests.length ? this.tests : this.enumerateTestFiles();
+        const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles();
 
         // tslint:disable-next-line:no-this-assignment
         const cls = this;

src/testRunner/parallel/host.ts

@@ -222,7 +222,7 @@ namespace Harness.Parallel.Host {
             console.log("Discovering runner-based tests...");
             const discoverStart = +(new Date());
             for (const runner of runners) {
-                for (const test of runner.enumerateTestFiles()) {
+                for (const test of runner.getTestFiles()) {
                     const file = typeof test === "string" ? test : test.file;
                     let size: number;
                     if (!perfData) {

src/testRunner/projectsRunner.ts

@@ -32,7 +32,11 @@ namespace project {
 
     export class ProjectRunner extends RunnerBase {
         public enumerateTestFiles() {
-            return this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true });
+            const all = this.enumerateFiles("tests/cases/project", /\.json$/, { recursive: true });
+            if (shards === 1) {
+                return all;
+            }
+            return all.filter((_val, idx) => idx % shards === (shardId - 1));
         }
 
         public kind(): TestRunnerKind {

src/testRunner/runner.ts

@@ -80,6 +80,8 @@ interface TestConfig {
     timeout?: number;
     keepFailed?: boolean;
     skipPercent?: number;
+    shardId?: number;
+    shards?: number;
 }
 
 interface TaskSet {
@@ -114,6 +116,12 @@ function handleTestConfig() {
         if (testConfig.skipPercent !== undefined) {
             skipPercent = testConfig.skipPercent;
         }
+        if (testConfig.shardId) {
+            shardId = testConfig.shardId;
+        }
+        if (testConfig.shards) {
+            shards = testConfig.shards;
+        }
 
         if (testConfig.stackTraceLimit === "full") {
             (<any>Error).stackTraceLimit = Infinity;
@@ -129,6 +137,9 @@ function handleTestConfig() {
 
         const runnerConfig = testConfig.runners || testConfig.test;
         if (runnerConfig && runnerConfig.length > 0) {
+            if (testConfig.runners) {
+                runUnitTests = runnerConfig.indexOf("unittest") !== -1;
+            }
             for (const option of runnerConfig) {
                 if (!option) {
                     continue;

src/testRunner/rwcRunner.ts

@@ -225,7 +225,7 @@ class RWCRunner extends RunnerBase {
      */
     public initializeTests(): void {
         // Read in and evaluate the test list
-        for (const test of this.tests && this.tests.length ? this.tests : this.enumerateTestFiles()) {
+        for (const test of this.tests && this.tests.length ? this.tests : this.getTestFiles()) {
             this.runTest(typeof test === "string" ? test : test.file);
         }
     }

src/testRunner/test262Runner.ts

@@ -97,7 +97,7 @@ class Test262BaselineRunner extends RunnerBase {
     public initializeTests() {
         // this will set up a series of describe/it blocks to run between the setup and cleanup phases
         if (this.tests.length === 0) {
-            const testFiles = this.enumerateTestFiles();
+            const testFiles = this.getTestFiles();
             testFiles.forEach(fn => {
                 this.runTest(fn);
             });