diff --git a/src/harness/compilerRunner.ts b/src/harness/compilerRunner.ts
index a04f77709ac..91a95e22c89 100644
--- a/src/harness/compilerRunner.ts
+++ b/src/harness/compilerRunner.ts
@@ -68,7 +68,7 @@ class CompilerBaselineRunner extends RunnerBase {
     public checkTestCodeOutput(fileName: string, test?: CompilerFileBasedTest) {
         if (test && test.configurations) {
             test.configurations.forEach(configuration => {
-                describe(`${this.testSuiteName} tests for ${fileName}${configuration && configuration.name ? ` (${configuration.name})` : ``}`, () => {
+                describe(`${this.testSuiteName} tests for ${fileName}${configuration ? ` (${Harness.getFileBasedTestConfigurationDescription(configuration)})` : ``}`, () => {
                     this.runSuite(fileName, test, configuration);
                 });
             });
@@ -82,7 +82,7 @@ class CompilerBaselineRunner extends RunnerBase {
         // Mocha holds onto the closure environment of the describe callback even after the test is done.
         // Everything declared here should be cleared out in the "after" callback.
         let compilerTest: CompilerTest | undefined;
-        before(() => { compilerTest = new CompilerTest(fileName, test && test.payload, configuration && configuration.settings); });
+        before(() => { compilerTest = new CompilerTest(fileName, test && test.payload, configuration); });
         it(`Correct errors for ${fileName}`, () => { compilerTest.verifyDiagnostics(); });
         it(`Correct module resolution tracing for ${fileName}`, () => { compilerTest.verifyModuleResolution(); });
         it(`Correct sourcemap content for ${fileName}`, () => { compilerTest.verifySourceMapRecord(); });
@@ -198,31 +198,7 @@ class CompilerTest {
         const rootDir = file.indexOf("conformance") === -1 ? "tests/cases/compiler/" : ts.getDirectoryPath(file) + "/";
         const payload = Harness.TestCaseParser.makeUnitsFromTest(content, file, rootDir);
         const settings = Harness.TestCaseParser.extractCompilerSettings(content);
-        const scriptTargets = CompilerTest._split(settings.target);
-        const moduleKinds = CompilerTest._split(settings.module);
-        if (scriptTargets.length <= 1 && moduleKinds.length <= 1) {
-            return { file, payload };
-        }
-
-        const configurations: Harness.FileBasedTestConfiguration[] = [];
-        for (const scriptTarget of scriptTargets) {
-            for (const moduleKind of moduleKinds) {
-                const settings: Record<string, any> = {};
-                let name = "";
-                if (moduleKinds.length > 1) {
-                    settings.module = moduleKind;
-                    name += `@module: ${moduleKind || "none"}`;
-                }
-                if (scriptTargets.length > 1) {
-                    settings.target = scriptTarget;
-                    if (name) name += ", ";
-                    name += `@target: ${scriptTarget || "none"}`;
-                }
-
-                configurations.push({ name, settings });
-            }
-        }
-
+        const configurations = Harness.getFileBasedTestConfigurations(settings, /*varyBy*/ ["module", "target"]);
         return { file, payload, configurations };
     }
 
@@ -291,11 +267,6 @@ class CompilerTest {
             this.toBeCompiled.concat(this.otherFiles).filter(file => !!this.result.program.getSourceFile(file.unitName)));
     }
 
-    private static _split(text: string) {
-        const entries = text && text.split(",").map(s => s.toLowerCase().trim()).filter(s => s.length > 0);
-        return entries && entries.length > 0 ? entries : [""];
-    }
-
     private makeUnitName(name: string, root: string) {
         const path = ts.toPath(name, root, ts.identity);
         const pathStart = ts.toPath(Harness.IO.getCurrentDirectory(), "", ts.identity);
diff --git a/src/harness/harness.ts b/src/harness/harness.ts
index 6b718f58841..28233b797da 100644
--- a/src/harness/harness.ts
+++ b/src/harness/harness.ts
@@ -499,16 +499,6 @@ namespace Utils {
 }
 
 namespace Harness {
-    export interface FileBasedTest {
-        file: string;
-        configurations?: FileBasedTestConfiguration[];
-    }
-
-    export interface FileBasedTestConfiguration {
-        name: string;
-        settings?: Record<string, string>;
-    }
-
     // tslint:disable-next-line:interface-name
     export interface IO {
         newLine(): string;
@@ -1783,6 +1773,74 @@ namespace Harness {
         }
     }
 
+    export interface FileBasedTest {
+        file: string;
+        configurations?: FileBasedTestConfiguration[];
+    }
+
+    export interface FileBasedTestConfiguration {
+        [key: string]: string;
+    }
+
+    function splitVaryBySettingValue(text: string): string[] | undefined {
+        if (!text) return undefined;
+        const entries = text.split(/,/).map(s => s.trim().toLowerCase()).filter(s => s.length > 0);
+        return entries && entries.length > 1 ? entries : undefined;
+    }
+
+    function computeFileBasedTestConfigurationVariations(configurations: FileBasedTestConfiguration[], variationState: FileBasedTestConfiguration, varyByEntries: [string, string[]][], offset: number) {
+        if (offset >= varyByEntries.length) {
+            // make a copy of the current variation state
+            configurations.push({ ...variationState });
+            return;
+        }
+
+        const [varyBy, entries] = varyByEntries[offset];
+        for (const entry of entries) {
+            // set or overwrite the variation, then compute the next variation
+            variationState[varyBy] = entry;
+            computeFileBasedTestConfigurationVariations(configurations, variationState, varyByEntries, offset + 1);
+        }
+    }
+
+    /**
+     * Compute FileBasedTestConfiguration variations based on a supplied list of variable settings.
+     */
+    export function getFileBasedTestConfigurations(settings: TestCaseParser.CompilerSettings, varyBy: string[]): FileBasedTestConfiguration[] | undefined {
+        let varyByEntries: [string, string[]][] | undefined;
+        for (const varyByKey of varyBy) {
+            if (ts.hasProperty(settings, varyByKey)) {
+                // we only consider variations when there are 2 or more variable entries.
+                const entries = splitVaryBySettingValue(settings[varyByKey]);
+                if (entries) {
+                    if (!varyByEntries) varyByEntries = [];
+                    varyByEntries.push([varyByKey, entries]);
+                }
+            }
+        }
+
+        if (!varyByEntries) return undefined;
+
+        const configurations: FileBasedTestConfiguration[] = [];
+        computeFileBasedTestConfigurationVariations(configurations, /*variationState*/ {}, varyByEntries, /*offset*/ 0);
+        return configurations;
+    }
+
+    /**
+     * Compute a description for this configuration based on its entries
+     */
+    export function getFileBasedTestConfigurationDescription(configuration: FileBasedTestConfiguration) {
+        let name = "";
+        if (configuration) {
+            const keys = Object.keys(configuration).sort();
+            for (const key of keys) {
+                if (name) name += ", ";
+                name += `@${key}: ${configuration[key]}`;
+            }
+        }
+        return name;
+    }
+
     export namespace TestCaseParser {
         /** all the necessary information to set the right compiler settings */
         export interface CompilerSettings {
diff --git a/tests/webTestServer.ts b/tests/webTestServer.ts
index 372a4bdffad..22f066bbf85 100644
--- a/tests/webTestServer.ts
+++ b/tests/webTestServer.ts
@@ -24,14 +24,13 @@ let browser = "IE";
 let grep: string | undefined;
 let verbose = false;
 
-interface FileTest {
+interface FileBasedTest {
     file: string;
-    configurations?: FileTestConfiguration[];
+    configurations?: FileBasedTestConfiguration[];
 }
 
-interface FileTestConfiguration {
-    name: string;
-    settings: Record<string, string>;
+interface FileBasedTestConfiguration {
+    [setting: string]: string;
 }
 
 function isFileSystemCaseSensitive(): boolean {
@@ -669,7 +668,7 @@ function handleApiEnumerateTestFiles(req: http.ServerRequest, res: http.ServerRe
         try {
             if (err) return sendInternalServerError(res, err);
             if (!content) return sendBadRequest(res);
-            const tests: (string | FileTest)[] = enumerateTestFiles(content);
+            const tests: (string | FileBasedTest)[] = enumerateTestFiles(content);
             return sendJson(res, /*statusCode*/ 200, tests);
         }
         catch (e) {
@@ -699,59 +698,62 @@ function enumerateTestFiles(runner: string) {
 // Regex for parsing options in the format "@Alpha: Value of any sort"
 const optionRegex = /^[\/]{2}\s*@(\w+)\s*:\s*([^\r\n]*)/gm;  // multiple matches on multiple lines
 
-function extractCompilerSettings(content: string): Record<string, any> {
-    const opts: Record<string, any> = {};
+function extractCompilerSettings(content: string): Record<string, string> {
+    const opts: Record<string, string> = {};
 
     let match: RegExpExecArray;
-    /* tslint:disable:no-null-keyword */
     while ((match = optionRegex.exec(content)) !== null) {
-    /* tslint:enable:no-null-keyword */
         opts[match[1]] = match[2].trim();
     }
 
     return opts;
 }
 
-function split(text: string) {
-    const entries = text && text.split(",").map(s => s.toLowerCase().trim()).filter(s => s.length > 0);
-    return entries && entries.length > 0 ? entries : [""];
+function splitVaryBySettingValue(text: string): string[] | undefined {
+    if (!text) return undefined;
+    const entries = text.split(/,/).map(s => s.trim().toLowerCase()).filter(s => s.length > 0);
+    return entries && entries.length > 1 ? entries : undefined;
 }
 
-function parseCompilerTestConfigurations(file: string): FileTest {
-    const content = fs.readFileSync(path.join(rootDir, file), "utf8");
-    const settings = extractCompilerSettings(content);
-    const scriptTargets = split(settings.target);
-    const moduleKinds = split(settings.module);
-    if (scriptTargets.length <= 1 && moduleKinds.length <= 1) {
-        return { file };
+function computeFileBasedTestConfigurationVariations(configurations: FileBasedTestConfiguration[], variationState: FileBasedTestConfiguration, varyByEntries: [string, string[]][], offset: number) {
+    if (offset >= varyByEntries.length) {
+        // make a copy of the current variation state
+        configurations.push({ ...variationState });
+        return;
     }
 
-    const configurations: FileTestConfiguration[] = [];
-    for (const scriptTarget of scriptTargets) {
-        for (const moduleKind of moduleKinds) {
-            const settings: Record<string, any> = {};
-            let name = "";
-            if (moduleKinds.length > 1) {
-                settings.module = moduleKind;
-                name += `@module: ${moduleKind || "none"}`;
+    const [varyBy, entries] = varyByEntries[offset];
+    for (const entry of entries) {
+        // set or overwrite the variation
+        variationState[varyBy] = entry;
+        computeFileBasedTestConfigurationVariations(configurations, variationState, varyByEntries, offset + 1);
+    }
+}
+
+function getFileBasedTestConfigurations(settings: Record<string, string>, varyBy: string[]): FileBasedTestConfiguration[] | undefined {
+    let varyByEntries: [string, string[]][] | undefined;
+    for (const varyByKey of varyBy) {
+        if (Object.prototype.hasOwnProperty.call(settings, varyByKey)) {
+            const entries = splitVaryBySettingValue(settings[varyByKey]);
+            if (entries) {
+                if (!varyByEntries) varyByEntries = [];
+                varyByEntries.push([varyByKey, entries]);
             }
-            if (scriptTargets.length > 1) {
-                settings.target = scriptTarget;
-                if (name) name += ", ";
-                name += `@target: ${scriptTarget || "none"}`;
-            }
-            configurations.push({ name, settings });
         }
     }
 
-    return { file, configurations };
+    if (!varyByEntries) return undefined;
+
+    const configurations: FileBasedTestConfiguration[] = [];
+    computeFileBasedTestConfigurationVariations(configurations, {}, varyByEntries, 0);
+    return configurations;
 }
 
-function parseProjectTestConfigurations(file: string): FileTest {
-    return { file, configurations: [
-        { name: `@module: commonjs`, settings: { module: "commonjs" } },
-        { name: `@module: amd`, settings: { module: "amd" } },
-    ] };
+function parseCompilerTestConfigurations(file: string): FileBasedTest {
+    const content = fs.readFileSync(path.join(rootDir, file), "utf8");
+    const settings = extractCompilerSettings(content);
+    const configurations = getFileBasedTestConfigurations(settings, ["module", "target"]);
+    return { file, configurations };
 }
 
 function handleApiListFiles(req: http.ServerRequest, res: http.ServerResponse) {