From 5f49357bf658e056674f16a356f63789478c68c1 Mon Sep 17 00:00:00 2001 From: Wesley Wigham Date: Tue, 19 Sep 2017 16:52:56 -0700 Subject: [PATCH] Fix unittest parallel reporting (#18583) * Some tests depended on late execution * Emulate mocha execution order * Polyfill a synchronous done to handle that one unittest * Accpept updates tsconfig baselines fixed by #18534 --- src/harness/parallel/worker.ts | 140 +++++++++++++----- .../tsconfig.json | 2 +- .../tsconfig.json | 2 +- .../tsconfig.json | 2 +- .../tsconfig.json | 2 +- .../tsconfig.json | 2 +- .../tsconfig.json | 2 +- .../tsconfig.json | 2 +- .../tsconfig.json | 2 +- 9 files changed, 114 insertions(+), 42 deletions(-) diff --git a/src/harness/parallel/worker.ts b/src/harness/parallel/worker.ts index 34f89d37f13..56f8fd43c58 100644 --- a/src/harness/parallel/worker.ts +++ b/src/harness/parallel/worker.ts @@ -1,52 +1,99 @@ namespace Harness.Parallel.Worker { let errors: ErrorInfo[] = []; let passing = 0; + let reportedUnitTests = false; + + type Executor = {name: string, callback: Function, kind: "suite" | "test"} | never; + function resetShimHarnessAndExecute(runner: RunnerBase) { - errors = []; - passing = 0; + if (reportedUnitTests) { + errors = []; + passing = 0; + testList.length = 0; + } + reportedUnitTests = true; runner.initializeTests(); + testList.forEach(({ name, callback, kind }) => executeCallback(name, callback, kind)); return { errors, passing }; } + + let beforeEachFunc: Function; + const namestack: string[] = []; + let testList: Executor[] = []; function shimMochaHarness() { (global as any).before = undefined; (global as any).after = undefined; (global as any).beforeEach = undefined; - let beforeEachFunc: Function; - describe = ((_name, callback) => { - const fakeContext: Mocha.ISuiteCallbackContext = { - retries() { return this; }, - slow() { return this; }, - timeout() { return this; }, - }; - (before as any) = (cb: Function) => cb(); - let afterFunc: Function; - (after as any) = (cb: Function) => afterFunc = cb; - const savedBeforeEach = beforeEachFunc; - (beforeEach as any) = (cb: Function) => beforeEachFunc = cb; - callback.call(fakeContext); - afterFunc && afterFunc(); - afterFunc = undefined; - beforeEachFunc = savedBeforeEach; + describe = ((name, callback) => { + testList.push({ name, callback, kind: "suite" }); }) as Mocha.IContextDefinition; it = ((name, callback) => { - const fakeContext: Mocha.ITestCallbackContext = { - skip() { return this; }, - timeout() { return this; }, - retries() { return this; }, - slow() { return this; }, - }; - // TODO: If we ever start using async test completions, polyfill the `done` parameter/promise return handling - if (beforeEachFunc) { - try { - beforeEachFunc(); - } - catch (error) { - errors.push({ error: error.message, stack: error.stack, name }); - return; - } + if (!testList) { + throw new Error("Tests must occur within a describe block"); } + testList.push({ name, callback, kind: "test" }); + }) as Mocha.ITestDefinition; + } + + function executeSuiteCallback(name: string, callback: Function) { + const fakeContext: Mocha.ISuiteCallbackContext = { + retries() { return this; }, + slow() { return this; }, + timeout() { return this; }, + }; + namestack.push(name); + let beforeFunc: Function; + (before as any) = (cb: Function) => beforeFunc = cb; + let afterFunc: Function; + (after as any) = (cb: Function) => afterFunc = cb; + const savedBeforeEach = beforeEachFunc; + (beforeEach as any) = (cb: Function) => beforeEachFunc = cb; + const savedTestList = testList; + + testList = []; + callback.call(fakeContext); + beforeFunc && beforeFunc(); + beforeFunc = undefined; + testList.forEach(({ name, callback, kind }) => executeCallback(name, callback, kind)); + testList.length = 0; + testList = savedTestList; + + afterFunc && afterFunc(); + afterFunc = undefined; + beforeEachFunc = savedBeforeEach; + namestack.pop(); + } + + function executeCallback(name: string, callback: Function, kind: "suite" | "test") { + if (kind === "suite") { + executeSuiteCallback(name, callback); + } + else { + executeTestCallback(name, callback); + } + } + + function executeTestCallback(name: string, callback: Function) { + const fakeContext: Mocha.ITestCallbackContext = { + skip() { return this; }, + timeout() { return this; }, + retries() { return this; }, + slow() { return this; }, + }; + name = [...namestack, name].join(" "); + if (beforeEachFunc) { try { + beforeEachFunc(); + } + catch (error) { + errors.push({ error: error.message, stack: error.stack, name }); + return; + } + } + if (callback.length === 0) { + try { + // TODO: If we ever start using async test completions, polyfill promise return handling callback.call(fakeContext); } catch (error) { @@ -54,7 +101,32 @@ namespace Harness.Parallel.Worker { return; } passing++; - }) as Mocha.ITestDefinition; + } + else { + // Uses `done` callback + let completed = false; + try { + callback.call(fakeContext, (err: any) => { + if (completed) { + throw new Error(`done() callback called multiple times; ensure it is only called once.`); + } + if (err) { + errors.push({ error: err.toString(), stack: "", name }); + } + else { + passing++; + } + completed = true; + }); + } + catch (error) { + errors.push({ error: error.message, stack: error.stack, name }); + return; + } + if (!completed) { + errors.push({ error: "Test completes asynchronously, which is unsupported by the parallel harness", stack: "", name }); + } + } } export function start() { diff --git a/tests/baselines/reference/tsConfig/Default initialized TSConfig/tsconfig.json b/tests/baselines/reference/tsConfig/Default initialized TSConfig/tsconfig.json index 0f5b2378468..a1bc2185da5 100644 --- a/tests/baselines/reference/tsConfig/Default initialized TSConfig/tsconfig.json +++ b/tests/baselines/reference/tsConfig/Default initialized TSConfig/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { /* Basic Options */ "target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */ - "module": "commonjs", /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ + "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ // "lib": [], /* Specify library files to be included in the compilation: */ // "allowJs": true, /* Allow javascript files to be compiled. */ // "checkJs": true, /* Report errors in .js files. */ diff --git a/tests/baselines/reference/tsConfig/Initialized TSConfig with boolean value compiler options/tsconfig.json b/tests/baselines/reference/tsConfig/Initialized TSConfig with boolean value compiler options/tsconfig.json index a545124a723..12ce62fdefb 100644 --- a/tests/baselines/reference/tsConfig/Initialized TSConfig with boolean value compiler options/tsconfig.json +++ b/tests/baselines/reference/tsConfig/Initialized TSConfig with boolean value compiler options/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { /* Basic Options */ "target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */ - "module": "commonjs", /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ + "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ // "lib": [], /* Specify library files to be included in the compilation: */ // "allowJs": true, /* Allow javascript files to be compiled. */ // "checkJs": true, /* Report errors in .js files. */ diff --git a/tests/baselines/reference/tsConfig/Initialized TSConfig with enum value compiler options/tsconfig.json b/tests/baselines/reference/tsConfig/Initialized TSConfig with enum value compiler options/tsconfig.json index b53ac2d8552..ea94f857fa4 100644 --- a/tests/baselines/reference/tsConfig/Initialized TSConfig with enum value compiler options/tsconfig.json +++ b/tests/baselines/reference/tsConfig/Initialized TSConfig with enum value compiler options/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { /* Basic Options */ "target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */ - "module": "commonjs", /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ + "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ // "lib": [], /* Specify library files to be included in the compilation: */ // "allowJs": true, /* Allow javascript files to be compiled. */ // "checkJs": true, /* Report errors in .js files. */ diff --git a/tests/baselines/reference/tsConfig/Initialized TSConfig with files options/tsconfig.json b/tests/baselines/reference/tsConfig/Initialized TSConfig with files options/tsconfig.json index 4e06e06d159..40181bc2553 100644 --- a/tests/baselines/reference/tsConfig/Initialized TSConfig with files options/tsconfig.json +++ b/tests/baselines/reference/tsConfig/Initialized TSConfig with files options/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { /* Basic Options */ "target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */ - "module": "commonjs", /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ + "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ // "lib": [], /* Specify library files to be included in the compilation: */ // "allowJs": true, /* Allow javascript files to be compiled. */ // "checkJs": true, /* Report errors in .js files. */ diff --git a/tests/baselines/reference/tsConfig/Initialized TSConfig with incorrect compiler option value/tsconfig.json b/tests/baselines/reference/tsConfig/Initialized TSConfig with incorrect compiler option value/tsconfig.json index 94808d89ed0..228c332c1e3 100644 --- a/tests/baselines/reference/tsConfig/Initialized TSConfig with incorrect compiler option value/tsconfig.json +++ b/tests/baselines/reference/tsConfig/Initialized TSConfig with incorrect compiler option value/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { /* Basic Options */ "target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */ - "module": "commonjs", /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ + "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ "lib": ["es5","es2015.promise"], /* Specify library files to be included in the compilation: */ // "allowJs": true, /* Allow javascript files to be compiled. */ // "checkJs": true, /* Report errors in .js files. */ diff --git a/tests/baselines/reference/tsConfig/Initialized TSConfig with incorrect compiler option/tsconfig.json b/tests/baselines/reference/tsConfig/Initialized TSConfig with incorrect compiler option/tsconfig.json index 0f5b2378468..a1bc2185da5 100644 --- a/tests/baselines/reference/tsConfig/Initialized TSConfig with incorrect compiler option/tsconfig.json +++ b/tests/baselines/reference/tsConfig/Initialized TSConfig with incorrect compiler option/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { /* Basic Options */ "target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */ - "module": "commonjs", /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ + "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ // "lib": [], /* Specify library files to be included in the compilation: */ // "allowJs": true, /* Allow javascript files to be compiled. */ // "checkJs": true, /* Report errors in .js files. */ diff --git a/tests/baselines/reference/tsConfig/Initialized TSConfig with list compiler options with enum value/tsconfig.json b/tests/baselines/reference/tsConfig/Initialized TSConfig with list compiler options with enum value/tsconfig.json index d165b0f2775..a3c6771965d 100644 --- a/tests/baselines/reference/tsConfig/Initialized TSConfig with list compiler options with enum value/tsconfig.json +++ b/tests/baselines/reference/tsConfig/Initialized TSConfig with list compiler options with enum value/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { /* Basic Options */ "target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */ - "module": "commonjs", /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ + "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ "lib": ["es5","es2015.core"], /* Specify library files to be included in the compilation: */ // "allowJs": true, /* Allow javascript files to be compiled. */ // "checkJs": true, /* Report errors in .js files. */ diff --git a/tests/baselines/reference/tsConfig/Initialized TSConfig with list compiler options/tsconfig.json b/tests/baselines/reference/tsConfig/Initialized TSConfig with list compiler options/tsconfig.json index 2a169b3aaaf..96b15f67792 100644 --- a/tests/baselines/reference/tsConfig/Initialized TSConfig with list compiler options/tsconfig.json +++ b/tests/baselines/reference/tsConfig/Initialized TSConfig with list compiler options/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { /* Basic Options */ "target": "es5", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNEXT'. */ - "module": "commonjs", /* Specify module code generation: 'none', commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ + "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ // "lib": [], /* Specify library files to be included in the compilation: */ // "allowJs": true, /* Allow javascript files to be compiled. */ // "checkJs": true, /* Report errors in .js files. */