From 6c6d9bf0910c45031a80592ede148faefa8192d3 Mon Sep 17 00:00:00 2001 From: Mohamed Hegazy Date: Mon, 17 Nov 2014 11:01:05 -0800 Subject: [PATCH] Add test262 test runner --- .gitignore | 2 + Jakefile | 10 ++++ src/harness/compilerRunner.ts | 2 +- src/harness/harness.ts | 20 ++------ src/harness/projectsRunner.ts | 2 +- src/harness/runner.ts | 7 ++- src/harness/rwcRunner.ts | 2 +- src/harness/test262Runner.ts | 87 +++++++++++++++++++++++++++++++++++ 8 files changed, 110 insertions(+), 22 deletions(-) create mode 100644 src/harness/test262Runner.ts diff --git a/.gitignore b/.gitignore index fb84462083c..2d740532d56 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ tests/cases/*/*/*.js.map tests/cases/*/*/*/*.js.map tests/cases/*/*/*/*/*.js.map tests/cases/rwc/* +tests/cases/test262/* tests/cases/perf/* !tests/cases/webharness/compilerToString.js test-args.txt @@ -19,6 +20,7 @@ tests/baselines/local/* tests/services/baselines/local/* tests/baselines/prototyping/local/* tests/baselines/rwc/* +tests/baselines/test262/* tests/services/baselines/prototyping/local/* tests/services/browser/typescriptServices.js scripts/processDiagnosticMessages.d.ts diff --git a/Jakefile b/Jakefile index 0d19f0d0ba3..f438c97d625 100644 --- a/Jakefile +++ b/Jakefile @@ -78,6 +78,7 @@ var harnessSources = [ "projectsRunner.ts", "loggedIO.ts", "rwcRunner.ts", + "test262Runner.ts", "runner.ts" ].map(function (f) { return path.join(harnessDirectory, f); @@ -347,6 +348,9 @@ var refBaseline = "tests/baselines/reference/"; var localRwcBaseline = "tests/baselines/rwc/local/"; var refRwcBaseline = "tests/baselines/rwc/reference/"; +var localTest262Baseline = "tests/baselines/test262/local/"; +var refTest262Baseline = "tests/baselines/test262/reference/"; + desc("Builds the test infrastructure using the built compiler"); task("tests", ["local", run].concat(libraryTargets)); @@ -515,6 +519,12 @@ task("baseline-accept-rwc", function() { fs.renameSync(localRwcBaseline, refRwcBaseline); }); +desc("Makes the most recent test262 test results the new baseline, overwriting the old baseline"); +task("baseline-accept-test262", function() { + jake.rmRf(refTest262Baseline); + fs.renameSync(localTest262Baseline, refTest262Baseline); +}); + // Webhost var webhostPath = "tests/webhost/webtsc.ts"; diff --git a/src/harness/compilerRunner.ts b/src/harness/compilerRunner.ts index 4ca33f92804..243191e919a 100644 --- a/src/harness/compilerRunner.ts +++ b/src/harness/compilerRunner.ts @@ -10,7 +10,7 @@ const enum CompilerTestType { } class CompilerBaselineRunner extends RunnerBase { - protected basePath = 'tests/cases'; + private basePath = 'tests/cases'; private errors: boolean; private emit: boolean; private decl: boolean; diff --git a/src/harness/harness.ts b/src/harness/harness.ts index 3bc635e2ab4..9b12bd85322 100644 --- a/src/harness/harness.ts +++ b/src/harness/harness.ts @@ -537,6 +537,8 @@ module Harness { export var defaultLibFileName = 'lib.d.ts'; export var defaultLibSourceFile = ts.createSourceFile(defaultLibFileName, IO.readFile(libFolder + 'lib.core.d.ts'), /*languageVersion*/ ts.ScriptTarget.Latest, /*version:*/ "0"); + export var defaultES6LibSourceFile = ts.createSourceFile(defaultLibFileName, IO.readFile(libFolder + 'lib.es6.d.ts'), /*languageVersion*/ ts.ScriptTarget.Latest, /*version:*/ "0"); + // Cache these between executions so we don't have to re-parse them for every test export var fourslashFilename = 'fourslash.ts'; @@ -579,9 +581,8 @@ module Harness { return fourslashSourceFile; } else { - var lib = defaultLibFileName; if (fn === defaultLibFileName) { - return defaultLibSourceFile; + return languageVersion === ts.ScriptTarget.ES6 ? defaultES6LibSourceFile : defaultLibSourceFile; } // Don't throw here -- the compiler might be looking for a test that actually doesn't exist as part of the TC return null; @@ -799,7 +800,6 @@ module Harness { checker.checkProgram(); var hasEarlyErrors = checker.hasEarlyErrors(); - // only emit if there weren't parse errors var emitResult: ts.EmitResult; if (!hadParseErrors && !hasEarlyErrors) { @@ -1009,20 +1009,6 @@ module Harness { sys.newLine + sys.newLine + outputLines.join('\r\n'); } - /* TODO: Delete? - export function makeDefaultCompilerSettings(options?: { useMinimalDefaultLib: boolean; noImplicitAny: boolean; }) { - var useMinimalDefaultLib = options ? options.useMinimalDefaultLib : true; - var noImplicitAny = options ? options.noImplicitAny : false; - var settings = new TypeScript.CompilationSettings(); - settings.codeGenTarget = TypeScript.LanguageVersion.EcmaScript5; - settings.moduleGenTarget = TypeScript.ModuleGenTarget.Synchronous; - settings.noLib = useMinimalDefaultLib; - settings.noResolve = false; - settings.noImplicitAny = noImplicitAny; - return settings; - } - */ - /** The harness' compiler instance used when tests are actually run. Reseting or changing settings of this compiler instance must be done within a test case (i.e., describe/it) */ var harnessCompiler: HarnessCompiler; diff --git a/src/harness/projectsRunner.ts b/src/harness/projectsRunner.ts index 797bc12a42b..35efbffbd68 100644 --- a/src/harness/projectsRunner.ts +++ b/src/harness/projectsRunner.ts @@ -169,7 +169,7 @@ class ProjectRunner extends RunnerBase { function getSourceFile(filename: string, languageVersion: ts.ScriptTarget): ts.SourceFile { var sourceFile: ts.SourceFile = undefined; if (filename === Harness.Compiler.defaultLibFileName) { - sourceFile = Harness.Compiler.defaultLibSourceFile; + sourceFile = languageVersion === ts.ScriptTarget.ES6 ? Harness.Compiler.defaultES6LibSourceFile : Harness.Compiler.defaultLibSourceFile; } else { var text = getSourceFileText(filename); diff --git a/src/harness/runner.ts b/src/harness/runner.ts index 38e23bb10c5..da37c02e224 100644 --- a/src/harness/runner.ts +++ b/src/harness/runner.ts @@ -13,9 +13,9 @@ // limitations under the License. // +/// /// -// TODO: re-enable -// /// +/// /// /// @@ -69,6 +69,9 @@ if (testConfigFile !== '') { case 'rwc': runners.push(new RWCRunner()); break; + case 'test262': + runners.push(new Test262BaselineRunner()); + break; case 'reverse': reverse = true; break; diff --git a/src/harness/rwcRunner.ts b/src/harness/rwcRunner.ts index bfb139fa719..df843ccf885 100644 --- a/src/harness/rwcRunner.ts +++ b/src/harness/rwcRunner.ts @@ -20,7 +20,7 @@ module RWC { } } - function collateOutputs(outputFiles: Harness.Compiler.GeneratedFile[], clean?: (s: string) => string) { + export function collateOutputs(outputFiles: Harness.Compiler.GeneratedFile[], clean?: (s: string) => string) { // Collect, test, and sort the filenames function cleanName(fn: string) { var lastSlash = ts.normalizeSlashes(fn).lastIndexOf('/'); diff --git a/src/harness/test262Runner.ts b/src/harness/test262Runner.ts new file mode 100644 index 00000000000..5b664276b5b --- /dev/null +++ b/src/harness/test262Runner.ts @@ -0,0 +1,87 @@ +/// +/// +/// + +class Test262BaselineRunner extends RunnerBase { + private static basePath = 'tests/cases/test262'; + private static helpersFilePath = 'tests/cases/test262-harness/helpers.d.ts'; + private static helperFile = { + unitName: Test262BaselineRunner.helpersFilePath, + content: Harness.IO.readFile(Test262BaselineRunner.helpersFilePath) + }; + private static testFileExtensionRegex = /\.js$/; + private static options: ts.CompilerOptions = { + allowNonTsExtensions: true, + target: ts.ScriptTarget.Latest, + module: ts.ModuleKind.CommonJS + }; + private static baselineOptions: Harness.Baseline.BaselineOptions = { Subfolder: 'test262' }; + + private runTest(filePath: string) { + describe('test262 test for ' + filePath, () => { + // Mocha holds onto the closure environment of the describe callback even after the test is done. + // Everything declared here should be cleared out in the "after" callback. + var testState: { + filename: string; + compilerResult: Harness.Compiler.CompilerResult; + inputFiles: { unitName: string; content: string }[]; + }; + + before(() => { + var content = Harness.IO.readFile(filePath); + var testFilename = ts.removeFileExtension(filePath).replace(/\//g, '_') + ".test"; + var testCaseContent = Harness.TestCaseParser.makeUnitsFromTest(content, testFilename); + + var inputFiles = testCaseContent.testUnitData.map(unit => { + return { unitName: Test262BaselineRunner.basePath + "/" + unit.name, content: unit.content }; + }); + + // Emit the results + testState = { + filename: testFilename, + inputFiles: inputFiles, + compilerResult: undefined, + }; + + Harness.Compiler.getCompiler().compileFiles([Test262BaselineRunner.helperFile].concat(inputFiles), /*otherFiles*/ [], compilerResult => { + testState.compilerResult = compilerResult; + }, /*settingsCallback*/ undefined, Test262BaselineRunner.options); + }); + + after(() => { + testState = undefined; + }); + + it('has the expected emitted code', () => { + Harness.Baseline.runBaseline('has the expected emitted code', testState.filename + '.output.js', () => { + var files = testState.compilerResult.files.filter(f=> f.fileName !== Test262BaselineRunner.helpersFilePath); + return RWC.collateOutputs(files, s => SyntacticCleaner.clean(s)); + }, false, Test262BaselineRunner.baselineOptions); + }); + + it('has the expected errors', () => { + Harness.Baseline.runBaseline('has the expected errors', testState.filename + '.errors.txt', () => { + var errors = testState.compilerResult.errors; + if (errors.length === 0) { + return null; + } + + return Harness.Compiler.getErrorBaseline(testState.inputFiles, errors); + }, false, Test262BaselineRunner.baselineOptions); + }); + }); + } + + public initializeTests() { + // this will set up a series of describe/it blocks to run between the setup and cleanup phases + if (this.tests.length === 0) { + var testFiles = this.enumerateFiles(Test262BaselineRunner.basePath, Test262BaselineRunner.testFileExtensionRegex, { recursive: true }); + testFiles.forEach(fn => { + this.runTest(ts.normalizePath(fn)); + }); + } + else { + this.tests.forEach(test => this.runTest(test)); + } + } +} \ No newline at end of file