diff --git a/Jakefile.js b/Jakefile.js
index 33ad46c8b58..cdf8e30efe0 100644
--- a/Jakefile.js
+++ b/Jakefile.js
@@ -566,7 +566,7 @@ task("runtests", ["tests", builtLocalDirectory], function() {
colors = process.env.colors || process.env.color
colors = colors ? ' --no-colors ' : ' --colors ';
tests = tests ? ' -g ' + tests : '';
- reporter = process.env.reporter || process.env.r || 'dot';
+ reporter = process.env.reporter || process.env.r || 'mocha-fivemat-progress-reporter';
// timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally
// default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
var cmd = host + " -R " + reporter + tests + colors + ' -t ' + testTimeout + ' ' + run;
diff --git a/package.json b/package.json
index 82ab734d6f1..70ada6fef15 100644
--- a/package.json
+++ b/package.json
@@ -34,6 +34,7 @@
"chai": "latest",
"browserify": "latest",
"istanbul": "latest",
+ "mocha-fivemat-progress-reporter": "latest",
"tslint": "latest"
},
"scripts": {
diff --git a/src/harness/compilerRunner.ts b/src/harness/compilerRunner.ts
index 1b399e1b706..39d4b053956 100644
--- a/src/harness/compilerRunner.ts
+++ b/src/harness/compilerRunner.ts
@@ -10,6 +10,7 @@ const enum CompilerTestType {
class CompilerBaselineRunner extends RunnerBase {
private basePath = 'tests/cases';
+ private testSuiteName: string;
private errors: boolean;
private emit: boolean;
private decl: boolean;
@@ -24,16 +25,17 @@ class CompilerBaselineRunner extends RunnerBase {
this.decl = true;
this.output = true;
if (testType === CompilerTestType.Conformance) {
- this.basePath += '/conformance';
+ this.testSuiteName = 'conformance';
}
else if (testType === CompilerTestType.Regressions) {
- this.basePath += '/compiler';
+ this.testSuiteName = 'compiler';
}
else if (testType === CompilerTestType.Test262) {
- this.basePath += '/test262';
+ this.testSuiteName = 'test262';
} else {
- this.basePath += '/compiler'; // default to this for historical reasons
+ this.testSuiteName = 'compiler'; // default to this for historical reasons
}
+ this.basePath += '/' + this.testSuiteName;
}
public checkTestCodeOutput(fileName: string) {
@@ -100,7 +102,7 @@ class CompilerBaselineRunner extends RunnerBase {
});
beforeEach(() => {
- /* The compiler doesn't handle certain flags flipping during a single compilation setting. Tests on these flags will need
+ /* The compiler doesn't handle certain flags flipping during a single compilation setting. Tests on these flags will need
a fresh compiler instance for themselves and then create a fresh one for the next test. Would be nice to get dev fixes
eventually to remove this limitation. */
for (var i = 0; i < tcSettings.length; ++i) {
@@ -261,19 +263,19 @@ class CompilerBaselineRunner extends RunnerBase {
// NEWTODO: Type baselines
if (result.errors.length === 0) {
- // The full walker simulates the types that you would get from doing a full
+ // The full walker simulates the types that you would get from doing a full
// compile. The pull walker simulates the types you get when you just do
// a type query for a random node (like how the LS would do it). Most of the
// time, these will be the same. However, occasionally, they can be different.
// Specifically, when the compiler internally depends on symbol IDs to order
- // things, then we may see different results because symbols can be created in a
+ // things, then we may see different results because symbols can be created in a
// different order with 'pull' operations, and thus can produce slightly differing
// output.
//
// For example, with a full type check, we may see a type outputed as: number | string
// But with a pull type check, we may see it as: string | number
//
- // These types are equivalent, but depend on what order the compiler observed
+ // These types are equivalent, but depend on what order the compiler observed
// certain parts of the program.
let allFiles = toBeCompiled.concat(otherFiles).filter(file => !!program.getSourceFile(file.unitName));
@@ -384,25 +386,27 @@ class CompilerBaselineRunner extends RunnerBase {
}
public initializeTests() {
- describe("Setup compiler for compiler baselines", () => {
- var harnessCompiler = Harness.Compiler.getCompiler();
- this.parseOptions();
- });
-
- // this will set up a series of describe/it blocks to run between the setup and cleanup phases
- if (this.tests.length === 0) {
- var testFiles = this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true });
- testFiles.forEach(fn => {
- fn = fn.replace(/\\/g, "/");
- this.checkTestCodeOutput(fn);
+ describe(this.testSuiteName + ' tests', () => {
+ describe("Setup compiler for compiler baselines", () => {
+ var harnessCompiler = Harness.Compiler.getCompiler();
+ this.parseOptions();
});
- }
- else {
- this.tests.forEach(test => this.checkTestCodeOutput(test));
- }
- describe("Cleanup after compiler baselines", () => {
- var harnessCompiler = Harness.Compiler.getCompiler();
+ // this will set up a series of describe/it blocks to run between the setup and cleanup phases
+ if (this.tests.length === 0) {
+ var testFiles = this.enumerateFiles(this.basePath, /\.tsx?$/, { recursive: true });
+ testFiles.forEach(fn => {
+ fn = fn.replace(/\\/g, "/");
+ this.checkTestCodeOutput(fn);
+ });
+ }
+ else {
+ this.tests.forEach(test => this.checkTestCodeOutput(test));
+ }
+
+ describe("Cleanup after compiler baselines", () => {
+ var harnessCompiler = Harness.Compiler.getCompiler();
+ });
});
}
@@ -434,4 +438,4 @@ class CompilerBaselineRunner extends RunnerBase {
}
}
}
-}
\ No newline at end of file
+}
diff --git a/src/harness/fourslashRunner.ts b/src/harness/fourslashRunner.ts
index 0db01c30f32..97a7f191ec9 100644
--- a/src/harness/fourslashRunner.ts
+++ b/src/harness/fourslashRunner.ts
@@ -2,7 +2,7 @@
///
///
-const enum FourSlashTestType {
+const enum FourSlashTestType {
Native,
Shims,
Server
@@ -35,70 +35,72 @@ class FourSlashRunner extends RunnerBase {
this.tests = this.enumerateFiles(this.basePath, /\.ts/i, { recursive: false });
}
- this.tests.forEach((fn: string) => {
- describe(fn, () => {
- fn = ts.normalizeSlashes(fn);
- var justName = fn.replace(/^.*[\\\/]/, '');
+ describe(this.testSuiteName + ' tests', () => {
+ this.tests.forEach((fn: string) => {
+ describe(fn, () => {
+ fn = ts.normalizeSlashes(fn);
+ var justName = fn.replace(/^.*[\\\/]/, '');
+
+ // Convert to relative path
+ var testIndex = fn.indexOf('tests/');
+ if (testIndex >= 0) fn = fn.substr(testIndex);
- // Convert to relative path
- var testIndex = fn.indexOf('tests/');
- if (testIndex >= 0) fn = fn.substr(testIndex);
+ if (justName && !justName.match(/fourslash\.ts$/i) && !justName.match(/\.d\.ts$/i)) {
+ it(this.testSuiteName + ' test ' + justName + ' runs correctly',() => {
+ FourSlash.runFourSlashTest(this.basePath, this.testType, fn);
+ });
+ }
+ });
+ });
- if (justName && !justName.match(/fourslash\.ts$/i) && !justName.match(/\.d\.ts$/i)) {
- it(this.testSuiteName + ' test ' + justName + ' runs correctly', () => {
- FourSlash.runFourSlashTest(this.basePath, this.testType, fn);
- });
+ describe('Generate Tao XML', () => {
+ var invalidReasons: any = {};
+ FourSlash.xmlData.forEach(xml => {
+ if (xml.invalidReason !== null) {
+ invalidReasons[xml.invalidReason] = (invalidReasons[xml.invalidReason] || 0) + 1;
+ }
+ });
+ var invalidReport: { reason: string; count: number }[] = [];
+ for (var reason in invalidReasons) {
+ if (invalidReasons.hasOwnProperty(reason)) {
+ invalidReport.push({ reason: reason, count: invalidReasons[reason] });
+ }
}
- });
- });
+ invalidReport.sort((lhs, rhs) => lhs.count > rhs.count ? -1 : lhs.count === rhs.count ? 0 : 1);
- describe('Generate Tao XML', () => {
- var invalidReasons: any = {};
- FourSlash.xmlData.forEach(xml => {
- if (xml.invalidReason !== null) {
- invalidReasons[xml.invalidReason] = (invalidReasons[xml.invalidReason] || 0) + 1;
- }
+ var lines: string[] = [];
+ lines.push('');
+ lines.push('');
+ lines.push(' ');
+ lines.push(' ');
+ lines.push(' ');
+ lines.push(' ');
+ FourSlash.xmlData.forEach(xml => {
+ if (xml.invalidReason !== null) {
+ lines.push('');
+ } else {
+ lines.push(' ');
+ xml.actions.forEach(action => {
+ lines.push(' ' + action);
+ });
+ lines.push(' ');
+ }
+ });
+ lines.push(' ');
+ lines.push(' ');
+ lines.push(' ');
+ lines.push(' ');
+ lines.push(' ');
+ lines.push(' ');
+ lines.push(' ');
+ lines.push(' ');
+ lines.push('');
+ Harness.IO.writeFile('built/local/fourslash.xml', lines.join('\r\n'));
});
- var invalidReport: { reason: string; count: number }[] = [];
- for (var reason in invalidReasons) {
- if (invalidReasons.hasOwnProperty(reason)) {
- invalidReport.push({ reason: reason, count: invalidReasons[reason] });
- }
- }
- invalidReport.sort((lhs, rhs) => lhs.count > rhs.count ? -1 : lhs.count === rhs.count ? 0 : 1);
-
- var lines: string[] = [];
- lines.push('');
- lines.push('');
- lines.push(' ');
- lines.push(' ');
- lines.push(' ');
- lines.push(' ');
- FourSlash.xmlData.forEach(xml => {
- if (xml.invalidReason !== null) {
- lines.push('');
- } else {
- lines.push(' ');
- xml.actions.forEach(action => {
- lines.push(' ' + action);
- });
- lines.push(' ');
- }
- });
- lines.push(' ');
- lines.push(' ');
- lines.push(' ');
- lines.push(' ');
- lines.push(' ');
- lines.push(' ');
- lines.push(' ');
- lines.push(' ');
- lines.push('');
- Harness.IO.writeFile('built/local/fourslash.xml', lines.join('\r\n'));
});
}
}
@@ -108,4 +110,4 @@ class GeneratedFourslashRunner extends FourSlashRunner {
super(testType);
this.basePath += '/generated/';
}
-}
\ No newline at end of file
+}
diff --git a/src/harness/projectsRunner.ts b/src/harness/projectsRunner.ts
index 5eb0016ff8d..01dc52308ff 100644
--- a/src/harness/projectsRunner.ts
+++ b/src/harness/projectsRunner.ts
@@ -89,7 +89,7 @@ class ProjectRunner extends RunnerBase {
}
// When test case output goes to tests/baselines/local/projectOutput/testCaseName/moduleKind/
- // We have these two separate locations because when comparing baselines the baseline verifier will delete the existing file
+ // We have these two separate locations because when comparing baselines the baseline verifier will delete the existing file
// so even if it was created by compiler in that location, the file will be deleted by verified before we can read it
// so lets keep these two locations separate
function getProjectOutputFolder(fileName: string, moduleKind: ts.ModuleKind) {
@@ -194,7 +194,7 @@ class ProjectRunner extends RunnerBase {
};
}
}
-
+
function batchCompilerProjectTestCase(moduleKind: ts.ModuleKind): BatchCompileProjectTestCaseResult{
var nonSubfolderDiskFiles = 0;
@@ -230,7 +230,7 @@ class ProjectRunner extends RunnerBase {
var diskRelativeName = ts.getRelativePathToDirectoryOrUrl(testCase.projectRoot, diskFileName,
getCurrentDirectory(), Harness.Compiler.getCanonicalFileName, /*isAbsolutePathAnUrl*/ false);
if (ts.isRootedDiskPath(diskRelativeName) || diskRelativeName.substr(0, 3) === "../") {
- // If the generated output file resides in the parent folder or is rooted path,
+ // If the generated output file resides in the parent folder or is rooted path,
// we need to instead create files that can live in the project reference folder
// but make sure extension of these files matches with the fileName the compiler asked to write
diskRelativeName = "diskFile" + nonSubfolderDiskFiles++ +
@@ -330,108 +330,110 @@ class ProjectRunner extends RunnerBase {
var name = 'Compiling project for ' + testCase.scenario + ': testcase ' + testCaseFileName;
- describe(name, () => {
- function verifyCompilerResults(moduleKind: ts.ModuleKind) {
- function getCompilerResolutionInfo() {
- var resolutionInfo: ProjectRunnerTestCaseResolutionInfo = {
- scenario: testCase.scenario,
- projectRoot: testCase.projectRoot,
- inputFiles: testCase.inputFiles,
- out: testCase.out,
- outDir: testCase.outDir,
- sourceMap: testCase.sourceMap,
- mapRoot: testCase.mapRoot,
- resolveMapRoot: testCase.resolveMapRoot,
- sourceRoot: testCase.sourceRoot,
- resolveSourceRoot: testCase.resolveSourceRoot,
- declaration: testCase.declaration,
- baselineCheck: testCase.baselineCheck,
- runTest: testCase.runTest,
- bug: testCase.bug,
- rootDir: testCase.rootDir,
- resolvedInputFiles: ts.map(compilerResult.program.getSourceFiles(), inputFile => inputFile.fileName),
- emittedFiles: ts.map(compilerResult.outputFiles, outputFile => outputFile.emittedFileName)
- };
+ describe('Projects tests', () => {
+ describe(name, () => {
+ function verifyCompilerResults(moduleKind: ts.ModuleKind) {
+ function getCompilerResolutionInfo() {
+ var resolutionInfo: ProjectRunnerTestCaseResolutionInfo = {
+ scenario: testCase.scenario,
+ projectRoot: testCase.projectRoot,
+ inputFiles: testCase.inputFiles,
+ out: testCase.out,
+ outDir: testCase.outDir,
+ sourceMap: testCase.sourceMap,
+ mapRoot: testCase.mapRoot,
+ resolveMapRoot: testCase.resolveMapRoot,
+ sourceRoot: testCase.sourceRoot,
+ resolveSourceRoot: testCase.resolveSourceRoot,
+ declaration: testCase.declaration,
+ baselineCheck: testCase.baselineCheck,
+ runTest: testCase.runTest,
+ bug: testCase.bug,
+ rootDir: testCase.rootDir,
+ resolvedInputFiles: ts.map(compilerResult.program.getSourceFiles(), inputFile => inputFile.fileName),
+ emittedFiles: ts.map(compilerResult.outputFiles, outputFile => outputFile.emittedFileName)
+ };
- return resolutionInfo;
- }
+ return resolutionInfo;
+ }
- var compilerResult: BatchCompileProjectTestCaseResult;
+ var compilerResult: BatchCompileProjectTestCaseResult;
- it(name + ": " + moduleNameToString(moduleKind) , () => {
- // Compile using node
- compilerResult = batchCompilerProjectTestCase(moduleKind);
- });
-
- it('Resolution information of (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
- Harness.Baseline.runBaseline('Resolution information of (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.json', () => {
- return JSON.stringify(getCompilerResolutionInfo(), undefined, " ");
+ it(name + ": " + moduleNameToString(moduleKind) , () => {
+ // Compile using node
+ compilerResult = batchCompilerProjectTestCase(moduleKind);
});
- });
-
- it('Errors for (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
- if (compilerResult.errors.length) {
- Harness.Baseline.runBaseline('Errors for (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.errors.txt', () => {
- return getErrorsBaseline(compilerResult);
+ it('Resolution information of (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
+ Harness.Baseline.runBaseline('Resolution information of (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.json', () => {
+ return JSON.stringify(getCompilerResolutionInfo(), undefined, " ");
});
- }
- });
+ });
- it('Baseline of emitted result (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
- if (testCase.baselineCheck) {
- ts.forEach(compilerResult.outputFiles, outputFile => {
-
- Harness.Baseline.runBaseline('Baseline of emitted result (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + outputFile.fileName, () => {
- try {
- return ts.sys.readFile(getProjectOutputFolder(outputFile.fileName, compilerResult.moduleKind));
- }
- catch (e) {
- return undefined;
- }
- });
- });
- }
- });
-
-
- it('SourceMapRecord for (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
- if (compilerResult.sourceMapData) {
- Harness.Baseline.runBaseline('SourceMapRecord for (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.sourcemap.txt', () => {
- return Harness.SourceMapRecoder.getSourceMapRecord(compilerResult.sourceMapData, compilerResult.program,
- ts.filter(compilerResult.outputFiles, outputFile => Harness.Compiler.isJS(outputFile.emittedFileName)));
- });
- }
- });
-
- // Verify that all the generated .d.ts files compile
-
- it('Errors in generated Dts files for (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
- if (!compilerResult.errors.length && testCase.declaration) {
- var dTsCompileResult = compileCompileDTsFiles(compilerResult);
- if (dTsCompileResult.errors.length) {
- Harness.Baseline.runBaseline('Errors in generated Dts files for (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.dts.errors.txt', () => {
- return getErrorsBaseline(dTsCompileResult);
+ it('Errors for (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
+ if (compilerResult.errors.length) {
+ Harness.Baseline.runBaseline('Errors for (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.errors.txt', () => {
+ return getErrorsBaseline(compilerResult);
});
}
- }
- });
+ });
+
+
+ it('Baseline of emitted result (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
+ if (testCase.baselineCheck) {
+ ts.forEach(compilerResult.outputFiles, outputFile => {
+
+ Harness.Baseline.runBaseline('Baseline of emitted result (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + outputFile.fileName, () => {
+ try {
+ return ts.sys.readFile(getProjectOutputFolder(outputFile.fileName, compilerResult.moduleKind));
+ }
+ catch (e) {
+ return undefined;
+ }
+ });
+ });
+ }
+ });
+
+
+ it('SourceMapRecord for (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
+ if (compilerResult.sourceMapData) {
+ Harness.Baseline.runBaseline('SourceMapRecord for (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.sourcemap.txt', () => {
+ return Harness.SourceMapRecoder.getSourceMapRecord(compilerResult.sourceMapData, compilerResult.program,
+ ts.filter(compilerResult.outputFiles, outputFile => Harness.Compiler.isJS(outputFile.emittedFileName)));
+ });
+ }
+ });
+
+ // Verify that all the generated .d.ts files compile
+
+ it('Errors in generated Dts files for (' + moduleNameToString(moduleKind) + '): ' + testCaseFileName, () => {
+ if (!compilerResult.errors.length && testCase.declaration) {
+ var dTsCompileResult = compileCompileDTsFiles(compilerResult);
+ if (dTsCompileResult.errors.length) {
+ Harness.Baseline.runBaseline('Errors in generated Dts files for (' + moduleNameToString(compilerResult.moduleKind) + '): ' + testCaseFileName, getBaselineFolder(compilerResult.moduleKind) + testCaseJustName + '.dts.errors.txt', () => {
+ return getErrorsBaseline(dTsCompileResult);
+ });
+ }
+ }
+ });
+ after(() => {
+ compilerResult = undefined;
+ });
+ }
+
+ verifyCompilerResults(ts.ModuleKind.CommonJS);
+ verifyCompilerResults(ts.ModuleKind.AMD);
+
after(() => {
- compilerResult = undefined;
+ // Mocha holds onto the closure environment of the describe callback even after the test is done.
+ // Therefore we have to clean out large objects after the test is done.
+ testCase = undefined;
+ testFileText = undefined;
+ testCaseJustName = undefined;
});
- }
-
- verifyCompilerResults(ts.ModuleKind.CommonJS);
- verifyCompilerResults(ts.ModuleKind.AMD);
-
- after(() => {
- // Mocha holds onto the closure environment of the describe callback even after the test is done.
- // Therefore we have to clean out large objects after the test is done.
- testCase = undefined;
- testFileText = undefined;
- testCaseJustName = undefined;
});
});
}
-}
\ No newline at end of file
+}