Add test262 test runner

.gitignore

@@ -11,6 +11,7 @@ tests/cases/*/*/*.js.map
 tests/cases/*/*/*/*.js.map
 tests/cases/*/*/*/*/*.js.map
 tests/cases/rwc/*
+tests/cases/test262/*
 tests/cases/perf/*
 !tests/cases/webharness/compilerToString.js
 test-args.txt
@@ -19,6 +20,7 @@ tests/baselines/local/*
 tests/services/baselines/local/*
 tests/baselines/prototyping/local/*
 tests/baselines/rwc/*
+tests/baselines/test262/*
 tests/services/baselines/prototyping/local/*
 tests/services/browser/typescriptServices.js
 scripts/processDiagnosticMessages.d.ts

Jakefile

@@ -78,6 +78,7 @@ var harnessSources = [
     "projectsRunner.ts",
     "loggedIO.ts",
     "rwcRunner.ts",
+    "test262Runner.ts",
     "runner.ts"
 ].map(function (f) {
     return path.join(harnessDirectory, f);
@@ -347,6 +348,9 @@ var refBaseline = "tests/baselines/reference/";
 var localRwcBaseline = "tests/baselines/rwc/local/";
 var refRwcBaseline = "tests/baselines/rwc/reference/";
 
+var localTest262Baseline = "tests/baselines/test262/local/";
+var refTest262Baseline = "tests/baselines/test262/reference/";
+
 desc("Builds the test infrastructure using the built compiler");
 task("tests", ["local", run].concat(libraryTargets));
 
@@ -515,6 +519,12 @@ task("baseline-accept-rwc", function() {
     fs.renameSync(localRwcBaseline, refRwcBaseline);
 });
 
+desc("Makes the most recent test262 test results the new baseline, overwriting the old baseline");
+task("baseline-accept-test262", function() {
+    jake.rmRf(refTest262Baseline);
+    fs.renameSync(localTest262Baseline, refTest262Baseline);
+});
+
 
 // Webhost
 var webhostPath = "tests/webhost/webtsc.ts";

src/harness/compilerRunner.ts

@@ -10,7 +10,7 @@ const enum CompilerTestType {
 }
 
 class CompilerBaselineRunner extends RunnerBase {
-    protected basePath = 'tests/cases';
+    private basePath = 'tests/cases';
     private errors: boolean;
     private emit: boolean;
     private decl: boolean;

src/harness/harness.ts

@@ -537,6 +537,8 @@ module Harness {
 
         export var defaultLibFileName = 'lib.d.ts';
         export var defaultLibSourceFile = ts.createSourceFile(defaultLibFileName, IO.readFile(libFolder + 'lib.core.d.ts'), /*languageVersion*/ ts.ScriptTarget.Latest, /*version:*/ "0");
+        export var defaultES6LibSourceFile = ts.createSourceFile(defaultLibFileName, IO.readFile(libFolder + 'lib.es6.d.ts'), /*languageVersion*/ ts.ScriptTarget.Latest, /*version:*/ "0");
+
 
         // Cache these between executions so we don't have to re-parse them for every test
         export var fourslashFilename = 'fourslash.ts';
@@ -579,9 +581,8 @@ module Harness {
                         return fourslashSourceFile;
                     }
                     else {
-                        var lib = defaultLibFileName;
                         if (fn === defaultLibFileName) {
-                            return defaultLibSourceFile;
+                            return languageVersion === ts.ScriptTarget.ES6 ? defaultES6LibSourceFile : defaultLibSourceFile;
                         }
                         // Don't throw here -- the compiler might be looking for a test that actually doesn't exist as part of the TC
                         return null;
@@ -799,7 +800,6 @@ module Harness {
                 checker.checkProgram();
 
                 var hasEarlyErrors = checker.hasEarlyErrors();
-
                 // only emit if there weren't parse errors
                 var emitResult: ts.EmitResult;
                 if (!hadParseErrors && !hasEarlyErrors) {
@@ -1009,20 +1009,6 @@ module Harness {
                 sys.newLine + sys.newLine + outputLines.join('\r\n');
         }
 
-        /* TODO: Delete?
-        export function makeDefaultCompilerSettings(options?: { useMinimalDefaultLib: boolean; noImplicitAny: boolean; }) {
-            var useMinimalDefaultLib = options ? options.useMinimalDefaultLib : true;
-            var noImplicitAny = options ? options.noImplicitAny : false;
-            var settings = new TypeScript.CompilationSettings();
-            settings.codeGenTarget = TypeScript.LanguageVersion.EcmaScript5;
-            settings.moduleGenTarget = TypeScript.ModuleGenTarget.Synchronous;
-            settings.noLib = useMinimalDefaultLib;
-            settings.noResolve = false;
-            settings.noImplicitAny = noImplicitAny;
-            return settings;
-        }
-        */
-
         /** The harness' compiler instance used when tests are actually run. Reseting or changing settings of this compiler instance must be done within a test case (i.e., describe/it) */
         var harnessCompiler: HarnessCompiler;
 

src/harness/projectsRunner.ts

@@ -169,7 +169,7 @@ class ProjectRunner extends RunnerBase {
             function getSourceFile(filename: string, languageVersion: ts.ScriptTarget): ts.SourceFile {
                 var sourceFile: ts.SourceFile = undefined;
                 if (filename === Harness.Compiler.defaultLibFileName) {
-                    sourceFile = Harness.Compiler.defaultLibSourceFile;
+                    sourceFile = languageVersion === ts.ScriptTarget.ES6 ? Harness.Compiler.defaultES6LibSourceFile : Harness.Compiler.defaultLibSourceFile;
                 }
                 else {
                     var text = getSourceFileText(filename);

src/harness/runner.ts

@@ -13,9 +13,9 @@
 // limitations under the License.
 //
 
+/// <reference path='test262Runner.ts' />
 /// <reference path='compilerRunner.ts' />
-// TODO: re-enable
-// ///<reference path='fourslashRunner.ts' />
+/// <reference path='fourslashRunner.ts' />
 /// <reference path='projectsRunner.ts' />
 /// <reference path='rwcRunner.ts' />
 
@@ -69,6 +69,9 @@ if (testConfigFile !== '') {
             case 'rwc':
                 runners.push(new RWCRunner());
                 break;
+            case 'test262':
+                runners.push(new Test262BaselineRunner());
+                break;
             case 'reverse':
                 reverse = true;
                 break;

src/harness/rwcRunner.ts

@@ -20,7 +20,7 @@ module RWC {
         }
     }
 
-    function collateOutputs(outputFiles: Harness.Compiler.GeneratedFile[], clean?: (s: string) => string) {
+    export function collateOutputs(outputFiles: Harness.Compiler.GeneratedFile[], clean?: (s: string) => string) {
         // Collect, test, and sort the filenames
         function cleanName(fn: string) {
             var lastSlash = ts.normalizeSlashes(fn).lastIndexOf('/');

src/harness/test262Runner.ts

@@ -0,0 +1,87 @@
+/// <reference path='harness.ts' />
+/// <reference path='runnerbase.ts' />
+/// <reference path='syntacticCleaner.ts' />
+
+class Test262BaselineRunner extends RunnerBase {
+    private static basePath = 'tests/cases/test262';
+    private static helpersFilePath = 'tests/cases/test262-harness/helpers.d.ts';
+    private static helperFile = {
+        unitName: Test262BaselineRunner.helpersFilePath,
+        content: Harness.IO.readFile(Test262BaselineRunner.helpersFilePath)
+    };
+    private static testFileExtensionRegex = /\.js$/;
+    private static options: ts.CompilerOptions = {
+        allowNonTsExtensions: true,
+        target: ts.ScriptTarget.Latest,
+        module: ts.ModuleKind.CommonJS
+    };
+    private static baselineOptions: Harness.Baseline.BaselineOptions = { Subfolder: 'test262' };
+
+    private runTest(filePath: string) {
+        describe('test262 test for ' + filePath, () => {
+            // Mocha holds onto the closure environment of the describe callback even after the test is done.
+            // Everything declared here should be cleared out in the "after" callback.
+            var testState: {
+                filename: string;
+                compilerResult: Harness.Compiler.CompilerResult;
+                inputFiles: { unitName: string; content: string }[];
+            };
+
+            before(() => {
+                var content = Harness.IO.readFile(filePath);
+                var testFilename = ts.removeFileExtension(filePath).replace(/\//g, '_') + ".test";
+                var testCaseContent = Harness.TestCaseParser.makeUnitsFromTest(content, testFilename);
+
+                var inputFiles = testCaseContent.testUnitData.map(unit => {
+                    return { unitName: Test262BaselineRunner.basePath + "/" + unit.name, content: unit.content };
+                });
+
+                // Emit the results
+                testState = {
+                    filename: testFilename,
+                    inputFiles: inputFiles,
+                    compilerResult: undefined,
+                };
+
+                Harness.Compiler.getCompiler().compileFiles([Test262BaselineRunner.helperFile].concat(inputFiles), /*otherFiles*/ [], compilerResult => {
+                    testState.compilerResult = compilerResult;
+                }, /*settingsCallback*/ undefined, Test262BaselineRunner.options);
+            });
+
+            after(() => {
+                testState = undefined;
+            });
+
+            it('has the expected emitted code', () => {
+                Harness.Baseline.runBaseline('has the expected emitted code', testState.filename + '.output.js', () => {
+                    var files = testState.compilerResult.files.filter(f=> f.fileName !== Test262BaselineRunner.helpersFilePath);
+                    return RWC.collateOutputs(files, s => SyntacticCleaner.clean(s));
+                }, false, Test262BaselineRunner.baselineOptions);
+            });
+
+            it('has the expected errors', () => {
+                Harness.Baseline.runBaseline('has the expected errors', testState.filename + '.errors.txt', () => {
+                    var errors = testState.compilerResult.errors;
+                    if (errors.length === 0) {
+                        return null;
+                    }
+
+                    return Harness.Compiler.getErrorBaseline(testState.inputFiles, errors);
+                }, false, Test262BaselineRunner.baselineOptions);
+            });
+        });
+    }
+
+    public initializeTests() {
+        // this will set up a series of describe/it blocks to run between the setup and cleanup phases
+        if (this.tests.length === 0) {
+            var testFiles = this.enumerateFiles(Test262BaselineRunner.basePath, Test262BaselineRunner.testFileExtensionRegex, { recursive: true });
+            testFiles.forEach(fn => {
+                this.runTest(ts.normalizePath(fn));
+            });
+        }
+        else {
+            this.tests.forEach(test => this.runTest(test));
+        }
+    }
+}