Include localized diagnostics (#18702)

* Add lcl files

* Add loclalization script

* Add localization build targets

* use async exists, and add assert

* Generate lcg file

* Add localize task to gulpFile

* Only run loclaize if the generated files neededs update. Also run localize as part of local

* Fix lint errors

* Linter love

* Respond to code review comments

.gitignore

@@ -19,11 +19,6 @@ tests/baselines/local/projectOutput/*
 tests/baselines/reference/testresults.tap
 tests/services/baselines/prototyping/local/*
 tests/services/browser/typescriptServices.js
-scripts/authors.js
-scripts/configureNightly.js
-scripts/processDiagnosticMessages.d.ts
-scripts/processDiagnosticMessages.js
-scripts/importDefinitelyTypedTests/importDefinitelyTypedTests.js
 src/harness/*.js
 src/compiler/diagnosticInformationMap.generated.ts
 src/compiler/diagnosticMessages.generated.json
@@ -43,7 +38,12 @@ scripts/run.bat
 scripts/word2md.js
 scripts/buildProtocol.js
 scripts/ior.js
-scripts/buildProtocol.js
+scripts/authors.js
+scripts/configureNightly.js
+scripts/processDiagnosticMessages.d.ts
+scripts/processDiagnosticMessages.js
+scripts/importDefinitelyTypedTests/importDefinitelyTypedTests.js
+scripts/generateLocalizedDiagnosticMessages.js
 scripts/*.js.map
 scripts/typings/
 coverage/

.npmignore

@@ -5,6 +5,7 @@ internal
 issue_template.md
 jenkins.sh
 lib/README.md
+lib/enu
 netci.groovy
 pull_request_template.md
 scripts
@@ -16,5 +17,5 @@ Jakefile.js
 .gitattributes
 .settings/
 .travis.yml
-.vscode/
+.vscode/
 test.config

Gulpfile.ts

@@ -87,6 +87,7 @@ const harnessDirectory = "src/harness/";
 const libraryDirectory = "src/lib/";
 const scriptsDirectory = "scripts/";
 const docDirectory = "doc/";
+const lclDirectory = "src/loc/lcl";
 
 const builtDirectory = "built/";
 const builtLocalDirectory = "built/local/";
@@ -367,6 +368,36 @@ gulp.task(builtGeneratedDiagnosticMessagesJSON, [diagnosticInfoMapTs], (done) =>
 
 gulp.task("generate-diagnostics", "Generates a diagnostic file in TypeScript based on an input JSON file", [diagnosticInfoMapTs]);
 
+//  Localize diagnostics script
+const generateLocalizedDiagnosticMessagesJs = path.join(scriptsDirectory, "generateLocalizedDiagnosticMessages.js");
+const generateLocalizedDiagnosticMessagesTs = path.join(scriptsDirectory, "generateLocalizedDiagnosticMessages.ts");
+
+gulp.task(generateLocalizedDiagnosticMessagesJs, /*help*/ false, [], () => {
+    const settings: tsc.Settings = getCompilerSettings({
+        target: "es5",
+        declaration: false,
+        removeComments: true,
+        noResolve: false,
+        stripInternal: false,
+        types: ["node", "xml2js"]
+    }, /*useBuiltCompiler*/ false);
+    return gulp.src(generateLocalizedDiagnosticMessagesTs)
+        .pipe(newer(generateLocalizedDiagnosticMessagesJs))
+        .pipe(sourcemaps.init())
+        .pipe(tsc(settings))
+        .pipe(sourcemaps.write("."))
+        .pipe(gulp.dest("."));
+});
+
+// Localize diagnostics
+const generatedLCGFile = path.join(builtLocalDirectory, "enu", "diagnosticMessages.generated.json.lcg");
+gulp.task(generatedLCGFile, [generateLocalizedDiagnosticMessagesJs, diagnosticInfoMapTs], (done) => {
+    if (fs.existsSync(builtLocalDirectory) && needsUpdate(generatedDiagnosticMessagesJSON, generatedLCGFile)) {
+        exec(host, [generateLocalizedDiagnosticMessagesJs, lclDirectory, builtLocalDirectory, generatedDiagnosticMessagesJSON], done, done);
+    }
+});
+
+gulp.task("localize", [generatedLCGFile]);
 
 const servicesFile = path.join(builtLocalDirectory, "typescriptServices.js");
 const standaloneDefinitionsFile = path.join(builtLocalDirectory, "typescriptServices.d.ts");
@@ -494,7 +525,7 @@ gulp.task(typesMapJson, /*help*/ false, [], () => {
 });
 
 gulp.task("lssl", "Builds language service server library", [tsserverLibraryFile]);
-gulp.task("local", "Builds the full compiler and services", [builtLocalCompiler, servicesFile, serverFile, builtGeneratedDiagnosticMessagesJSON, tsserverLibraryFile]);
+gulp.task("local", "Builds the full compiler and services", [builtLocalCompiler, servicesFile, serverFile, builtGeneratedDiagnosticMessagesJSON, tsserverLibraryFile, "localize"]);
 gulp.task("tsc", "Builds only the compiler", [builtLocalCompiler]);
 
 // Generate Markdown spec
@@ -536,15 +567,16 @@ gulp.task("dontUseDebugMode", /*help*/ false, [], (done) => { useDebugMode = fal
 
 gulp.task("VerifyLKG", /*help*/ false, [], () => {
     const expectedFiles = [builtLocalCompiler, servicesFile, serverFile, nodePackageFile, nodeDefinitionsFile, standaloneDefinitionsFile, tsserverLibraryFile, tsserverLibraryDefinitionFile, typingsInstallerJs, cancellationTokenJs].concat(libraryTargets);
-    const missingFiles = expectedFiles.filter(function(f) {
-        return !fs.existsSync(f);
-    });
+    const missingFiles = expectedFiles.
+        concat(fs.readdirSync(lclDirectory).map(function (d) { return path.join(builtLocalDirectory, d, "diagnosticMessages.generated.json"); })).
+        concat(generatedLCGFile).
+        filter(f => !fs.existsSync(f));
     if (missingFiles.length > 0) {
         throw new Error("Cannot replace the LKG unless all built targets are present in directory " + builtLocalDirectory +
             ". The following files are missing:\n" + missingFiles.join("\n"));
     }
     // Copy all the targets into the LKG directory
-    return gulp.src(expectedFiles).pipe(gulp.dest(LKGDirectory));
+    return gulp.src([...expectedFiles, path.join(builtLocalDirectory, "**"), `!${path.join(builtLocalDirectory, "tslint")}`, `!${path.join(builtLocalDirectory, "*.*")}`]).pipe(gulp.dest(LKGDirectory));
 });
 
 gulp.task("LKGInternal", /*help*/ false, ["lib", "local"]);
@@ -1051,7 +1083,7 @@ gulp.task("lint", "Runs tslint on the compiler sources. Optional arguments are:
     const fileMatcher = cmdLineOptions["files"];
     const files = fileMatcher
         ? `src/**/${fileMatcher}`
-        : "Gulpfile.ts 'scripts/tslint/**/*.ts' 'src/**/*.ts' --exclude src/lib/es5.d.ts --exclude 'src/lib/*.generated.d.ts'";
+        : "Gulpfile.ts 'scripts/generateLocalizedDiagnosticMessages.ts' scripts/tslint/**/*.ts' 'src/**/*.ts' --exclude src/lib/es5.d.ts --exclude 'src/lib/*.generated.d.ts'";
     const cmd = `node node_modules/tslint/bin/tslint ${files} --formatters-dir ./built/local/tslint/formatters --format autolinkableStylish`;
     console.log("Linting: " + cmd);
     child_process.execSync(cmd, { stdio: [0, 1, 2] });

Jakefile.js

@@ -17,6 +17,7 @@ var libraryDirectory = "src/lib/";
 var scriptsDirectory = "scripts/";
 var unittestsDirectory = "src/harness/unittests/";
 var docDirectory = "doc/";
+var lclDirectory = "src/loc/lcl";
 
 var builtDirectory = "built/";
 var builtLocalDirectory = "built/local/";
@@ -427,7 +428,40 @@ compileFile(processDiagnosticMessagesJs,
     [processDiagnosticMessagesTs],
     [processDiagnosticMessagesTs],
     [],
-            /*useBuiltCompiler*/ false);
+    /*useBuiltCompiler*/ false);
+
+// Localize diagnostics script
+var generateLocalizedDiagnosticMessagesJs = path.join(scriptsDirectory, "generateLocalizedDiagnosticMessages.js");
+var generateLocalizedDiagnosticMessagesTs = path.join(scriptsDirectory, "generateLocalizedDiagnosticMessages.ts");
+
+file(generateLocalizedDiagnosticMessagesTs);
+
+compileFile(generateLocalizedDiagnosticMessagesJs,
+    [generateLocalizedDiagnosticMessagesTs],
+    [generateLocalizedDiagnosticMessagesTs],
+    [],
+    /*useBuiltCompiler*/ false, { noOutFile: true, types: ["node", "xml2js"] });
+
+// Localize diagnostics
+var generatedLCGFile = path.join(builtLocalDirectory, "enu", "diagnosticMessages.generated.json.lcg");
+file(generatedLCGFile, [generateLocalizedDiagnosticMessagesJs, diagnosticInfoMapTs, generatedDiagnosticMessagesJSON], function () {
+    var cmd = host + " " + generateLocalizedDiagnosticMessagesJs + " " + lclDirectory + " " + builtLocalDirectory + " " + generatedDiagnosticMessagesJSON;
+    console.log(cmd);
+    var ex = jake.createExec([cmd]);
+    // Add listeners for output and error
+    ex.addListener("stdout", function (output) {
+        process.stdout.write(output);
+    });
+    ex.addListener("stderr", function (error) {
+        process.stderr.write(error);
+    });
+    ex.addListener("cmdEnd", function () {
+        complete();
+    });
+    ex.run();
+}, { async: true });
+
+task("localize", [generatedLCGFile]);
 
 var buildProtocolTs = path.join(scriptsDirectory, "buildProtocol.ts");
 var buildProtocolJs = path.join(scriptsDirectory, "buildProtocol.js");
@@ -644,7 +678,7 @@ task("build-fold-end", [], function () {
 
 // Local target to build the compiler and services
 desc("Builds the full compiler and services");
-task("local", ["build-fold-start", "generate-diagnostics", "lib", tscFile, servicesFile, nodeDefinitionsFile, serverFile, buildProtocolDts, builtGeneratedDiagnosticMessagesJSON, "lssl", "build-fold-end"]);
+task("local", ["build-fold-start", "generate-diagnostics", "lib", tscFile, servicesFile, nodeDefinitionsFile, serverFile, buildProtocolDts, builtGeneratedDiagnosticMessagesJSON, "lssl", "localize", "build-fold-end"]);
 
 // Local target to build only tsc.js
 desc("Builds only the compiler");
@@ -699,7 +733,10 @@ task("generate-spec", [specMd]);
 // Makes a new LKG. This target does not build anything, but errors if not all the outputs are present in the built/local directory
 desc("Makes a new LKG out of the built js files");
 task("LKG", ["clean", "release", "local"].concat(libraryTargets), function () {
-    var expectedFiles = [tscFile, servicesFile, serverFile, nodePackageFile, nodeDefinitionsFile, standaloneDefinitionsFile, tsserverLibraryFile, tsserverLibraryDefinitionFile, cancellationTokenFile, typingsInstallerFile, buildProtocolDts, watchGuardFile].concat(libraryTargets);
+    var expectedFiles = [tscFile, servicesFile, serverFile, nodePackageFile, nodeDefinitionsFile, standaloneDefinitionsFile, tsserverLibraryFile, tsserverLibraryDefinitionFile, cancellationTokenFile, typingsInstallerFile, buildProtocolDts, watchGuardFile].
+        concat(libraryTargets).
+        concat(fs.readdirSync(lclDirectory).map(function (d) { return path.join(builtLocalDirectory, d) })).
+        concat(path.dirname(generatedLCGFile));
     var missingFiles = expectedFiles.filter(function (f) {
         return !fs.existsSync(f);
     });
@@ -1229,7 +1266,7 @@ task("lint", ["build-rules"], () => {
     const fileMatcher = process.env.f || process.env.file || process.env.files;
     const files = fileMatcher
         ? `src/**/${fileMatcher}`
-        : "Gulpfile.ts 'scripts/tslint/**/*.ts' 'src/**/*.ts' --exclude src/lib/es5.d.ts --exclude 'src/lib/*.generated.d.ts'";
+        : "Gulpfile.ts 'scripts/generateLocalizedDiagnosticMessages.ts' 'scripts/tslint/**/*.ts' 'src/**/*.ts' --exclude src/lib/es5.d.ts --exclude 'src/lib/*.generated.d.ts'";
     const cmd = `node node_modules/tslint/bin/tslint ${files} --formatters-dir ./built/local/tslint/formatters --format autolinkableStylish`;
     console.log("Linting: " + cmd);
     jake.exec([cmd], { interactive: true }, () => {

package.json

@@ -49,6 +49,8 @@
         "@types/q": "latest",
         "@types/run-sequence": "latest",
         "@types/through2": "latest",
+        "@types/xml2js": "^0.4.0",
+        "xml2js": "^0.4.19",
         "browser-resolve": "^1.11.2",
         "browserify": "latest",
         "chai": "latest",

scripts/generateLocalizedDiagnosticMessages.ts

@@ -0,0 +1,139 @@
+import * as fs from "fs";
+import * as path from "path";
+import * as xml2js from "xml2js";
+
+function main(): void {
+    const args = process.argv.slice(2);
+    if (args.length !== 3) {
+        console.log("Usage:");
+        console.log("\tnode generateLocalizedDiagnosticMessages.js <lcl source directory> <output directory> <generated diagnostics map file>");
+        return;
+    }
+
+    const inputPath = args[0];
+    const outputPath = args[1];
+    const diagnosticsMapFilePath = args[2];
+
+    // generate the lcg file for enu
+    generateLCGFile();
+
+    // generate other langs
+    fs.readdir(inputPath, (err, files) => {
+        handleError(err);
+        files.forEach(visitDirectory);
+    });
+
+    return;
+
+    function visitDirectory(name: string) {
+        const inputFilePath = path.join(inputPath, name, "diagnosticMessages", "diagnosticMessages.generated.json.lcl");
+        const outputFilePath = path.join(outputPath, name, "diagnosticMessages.generated.json");
+        fs.readFile(inputFilePath, (err, data) => {
+            handleError(err);
+            xml2js.parseString(data.toString(), (err, result) => {
+                handleError(err);
+                writeFile(outputFilePath, xmlObjectToString(result));
+            });
+        });
+    }
+
+    function handleError(err: null | object) {
+        if (err) {
+            console.error(err);
+            process.exit(1);
+        }
+    }
+
+    function xmlObjectToString(o: any) {
+        const out: any = {};
+        for (const item of o["LCX"]["Item"][0]["Item"][0]["Item"]) {
+            let ItemId = item["$"]["ItemId"];
+            let Val = item["Str"][0]["Tgt"] ? item["Str"][0]["Tgt"][0]["Val"][0] : item["Str"][0]["Val"][0];
+
+            if (typeof ItemId !== "string" || typeof Val !== "string") {
+                console.error("Unexpected XML file structure");
+                process.exit(1);
+            }
+
+            if (ItemId.charAt(0) === ";") {
+                ItemId = ItemId.slice(1); // remove leading semicolon
+            }
+
+            Val = Val.replace(/]5D;/, "]"); // unescape `]`
+            out[ItemId] = Val;
+        }
+        return JSON.stringify(out, undefined, 2);
+    }
+
+
+    function ensureDirectoryExists(directoryPath: string, action: () => void) {
+        fs.exists(directoryPath, exists => {
+            if (!exists) {
+                const basePath = path.dirname(directoryPath);
+                if (basePath !== directoryPath) {
+                    return ensureDirectoryExists(basePath, () => fs.mkdir(directoryPath, action));
+                }
+            }
+            action();
+        });
+    }
+
+    function writeFile(fileName: string, contents: string) {
+        ensureDirectoryExists(path.dirname(fileName), () => {
+            fs.writeFile(fileName, contents, handleError);
+        });
+    }
+
+    function objectToList(o: Record<string, string>) {
+        const list: { key: string, value: string }[] = [];
+        for (const key in o) {
+            list.push({ key, value: o[key] });
+        }
+        return list;
+    }
+
+    function generateLCGFile() {
+        return fs.readFile(diagnosticsMapFilePath, (err, data) => {
+            handleError(err);
+            writeFile(
+                path.join(outputPath, "enu", "diagnosticMessages.generated.json.lcg"),
+                getLCGFileXML(
+                    objectToList(JSON.parse(data.toString()))
+                        .sort((a, b) => a.key > b.key ? 1 : -1)  // lcg sorted by property keys
+                        .reduce((s, { key, value }) => s + getItemXML(key, value), "")
+                ));
+        });
+
+        function getItemXML(key: string, value: string) {
+            // escape entrt value
+            value = value.replace(/]/, "]5D;");
+
+            return `
+            <Item ItemId=";${key}" ItemType="0" PsrId="306" Leaf="true">
+              <Str Cat="Text">
+                <Val><![CDATA[${value}]]></Val>
+              </Str>
+              <Disp Icon="Str" />
+            </Item>`;
+        }
+
+        function getLCGFileXML(items: string) {
+            return `<?xml version="1.0" encoding="utf-8"?>
+<LCX SchemaVersion="6.0" Name="diagnosticMessages.generated.json" PsrId="306" FileType="1" SrcCul="en-US" xmlns="http://schemas.microsoft.com/locstudio/2006/6/lcx">
+  <OwnedComments>
+    <Cmt Name="Dev" />
+    <Cmt Name="LcxAdmin" />
+    <Cmt Name="Rccx" />
+  </OwnedComments>
+  <Item ItemId=";String Table" ItemType="0" PsrId="306" Leaf="false">
+    <Disp Icon="Expand" Expand="true" Disp="true" LocTbl="false" />
+    <Item ItemId=";Strings" ItemType="0" PsrId="306" Leaf="false">
+      <Disp Icon="Str" Disp="true" LocTbl="false" />${items}
+    </Item>
+  </Item>
+</LCX>`;
+        }
+    }
+}
+
+main();