From bb1e7117f1f84288aa92d15c30d2ed2bc6b483b3 Mon Sep 17 00:00:00 2001 From: Michel Kaporin Date: Thu, 2 Mar 2017 21:53:02 +0100 Subject: [PATCH 1/5] Implemented transifex push and pull for translations together with json->xlf->json parsing. --- build/gulpfile.vscode.js | 33 ++ build/lib/i18n.js | 666 ++++++++++++++++++++++++++ build/lib/i18n.ts | 742 ++++++++++++++++++++++++++++- extensions/typescript/package.json | 2 +- 4 files changed, 1441 insertions(+), 2 deletions(-) diff --git a/build/gulpfile.vscode.js b/build/gulpfile.vscode.js index 61b1db21fde..989d1a78b72 100644 --- a/build/gulpfile.vscode.js +++ b/build/gulpfile.vscode.js @@ -29,6 +29,7 @@ const packageJson = require('../package.json'); const product = require('../product.json'); const shrinkwrap = require('../npm-shrinkwrap.json'); const crypto = require('crypto'); +const i18n = require('./lib/i18n'); const dependencies = Object.keys(shrinkwrap.dependencies) .concat(Array.isArray(product.extraNodeModules) ? product.extraNodeModules : []); // additional dependencies from our product configuration @@ -339,6 +340,38 @@ gulp.task('vscode-linux-ia32-min', ['minify-vscode', 'clean-vscode-linux-ia32'], gulp.task('vscode-linux-x64-min', ['minify-vscode', 'clean-vscode-linux-x64'], packageTask('linux', 'x64', { minified: true })); gulp.task('vscode-linux-arm-min', ['minify-vscode', 'clean-vscode-linux-arm'], packageTask('linux', 'arm', { minified: true })); +const apiUrl = process.env.TRANSIFEX_API_URL; +const apiName = process.env.TRANSIFEX_API_NAME; +const apiToken = process.env.TRANSIFEX_API_TOKEN; + +gulp.task('vscode-translations-update', function() { + const pathToMetadata = './out-vscode/nls.metadata.json'; + const pathToExtensions = './extensions/**/*.nls.json'; + const pathToSetup = 'build/win32/**/{Default.isl,messages.en.isl}'; + + gulp.src(pathToMetadata) + .pipe(i18n.prepareXlfFiles()) + .pipe(i18n.pushXlfFiles(apiUrl, apiName, apiToken)); + + gulp.src(pathToSetup) + .pipe(i18n.prepareXlfFiles()) + .pipe(i18n.pushXlfFiles(apiUrl, apiName, apiToken)); + + return gulp.src(pathToExtensions) + .pipe(i18n.prepareXlfFiles('vscode-extensions')) + .pipe(i18n.pushXlfFiles(apiUrl, apiName, apiToken)); +}); + +gulp.task('vscode-translations-pull', function() { + i18n.pullXlfFiles('vscode-editor-workbench', apiUrl, apiName, apiToken) + .pipe(i18n.prepareJsonFiles()) + .pipe(vfs.dest('./i18n')); + + return i18n.pullXlfFiles('vscode-extensions', apiUrl, apiName, apiToken) + .pipe(i18n.prepareJsonFiles()) + .pipe(vfs.dest('./i18n')); +}); + // Sourcemaps gulp.task('upload-vscode-sourcemaps', ['minify-vscode'], () => { diff --git a/build/lib/i18n.js b/build/lib/i18n.js index 7dcf0a0b763..a8256d1ad37 100644 --- a/build/lib/i18n.js +++ b/build/lib/i18n.js @@ -10,6 +10,13 @@ var event_stream_1 = require("event-stream"); var File = require("vinyl"); var Is = require("is"); var util = require('gulp-util'); +var xml2js = require("xml2js"); +var glob = require("glob"); +var util = require('gulp-util'); +var request = require('request'); +var es = require('event-stream'); +var iconv = require('iconv-lite'); + function log(message) { var rest = []; for (var _i = 1; _i < arguments.length; _i++) { @@ -37,6 +44,174 @@ var BundledFormat; } BundledFormat.is = is; })(BundledFormat || (BundledFormat = {})); + +var PackageJsonFormat; +(function (PackageJsonFormat) { + function is(value) { + if (Is.undef(value) || !Is.object(value)) { + return false; + } + return Object.keys(value).every(function (key) { + var element = value[key]; + return Is.string(element) || (Is.object(element) && Is.defined(element.message) && Is.defined(element.comment)); + }); + } + PackageJsonFormat.is = is; +})(PackageJsonFormat || (PackageJsonFormat = {})); +var ModuleJsonFormat; +(function (ModuleJsonFormat) { + function is(value) { + var candidate = value; + return Is.defined(candidate) + && Is.array(candidate.messages) && candidate.messages.every(function (message) { return Is.string(message); }) + && Is.array(candidate.keys) && candidate.keys.every(function (key) { return Is.string(key) || LocalizeInfo.is(key); }); + } + ModuleJsonFormat.is = is; +})(ModuleJsonFormat || (ModuleJsonFormat = {})); +var Line = (function () { + function Line(indent) { + if (indent === void 0) { indent = 0; } + this.indent = indent; + this.buffer = []; + if (indent > 0) { + this.buffer.push(new Array(indent + 1).join(' ')); + } + } + Line.prototype.append = function (value) { + this.buffer.push(value); + return this; + }; + Line.prototype.toString = function () { + return this.buffer.join(''); + }; + return Line; +}()); +exports.Line = Line; +var TextModel = (function () { + function TextModel(contents) { + this._lines = contents.split(/\r\n|\r|\n/); + } + Object.defineProperty(TextModel.prototype, "lines", { + get: function () { + return this._lines; + }, + enumerable: true, + configurable: true + }); + return TextModel; +}()); +var XLF = (function () { + function XLF(project) { + this.project = project; + this.buffer = []; + this.files = Object.create(null); + } + XLF.prototype.toString = function () { + this.appendHeader(); + for (var file in this.files) { + this.appendNewLine("", 2); + for (var _i = 0, _a = this.files[file]; _i < _a.length; _i++) { + var item = _a[_i]; + this.addStringItem(item); + } + this.appendNewLine('', 2); + } + this.appendFooter(); + return this.buffer.join('\r\n'); + }; + XLF.prototype.addFile = function (original, keys, messages) { + this.files[original] = []; + var existingKeys = []; + for (var _i = 0, keys_1 = keys; _i < keys_1.length; _i++) { + var key = keys_1[_i]; + // Ignore duplicate keys because Transifex does not populate those with translated values. + if (existingKeys.indexOf(key) !== -1) { + continue; + } + existingKeys.push(key); + var message = encodeEntities(messages[keys.indexOf(key)]); + var comment = undefined; + // Check if the message contains description (if so, it becomes an object type in JSON) + if (Is.string(key)) { + this.files[original].push({ id: key, message: message, comment: comment }); + } + else { + if (key['comment'] && key['comment'].length > 0) { + comment = key['comment'].map(function (comment) { return encodeEntities(comment); }).join('\r\n'); + } + this.files[original].push({ id: key['key'], message: message, comment: comment }); + } + } + }; + XLF.prototype.addStringItem = function (item) { + if (!item.id || !item.message) { + //throw new Error('No item ID or value specified.'); + } + this.appendNewLine("", 4); + this.appendNewLine("" + encodeEntities(item.message) + "", 6); + if (item.comment) { + this.appendNewLine("" + item.comment + "", 6); + } + this.appendNewLine('', 4); + }; + XLF.prototype.appendHeader = function () { + this.appendNewLine('', 0); + this.appendNewLine('', 0); + }; + XLF.prototype.appendFooter = function () { + this.appendNewLine('', 0); + }; + XLF.prototype.appendNewLine = function (content, indent) { + var line = new Line(indent); + line.append(content); + this.buffer.push(line.toString()); + }; + return XLF; +}()); +XLF.parse = function (xlfString) { + return new Promise(function (resolve, reject) { + var parser = new xml2js.Parser(); + var files = []; + parser.parseString(xlfString, function (err, result) { + if (err) { + reject("Failed to parse XLIFF string. " + err); + } + var fileNodes = result['xliff']['file']; + if (!fileNodes) { + reject('XLIFF file does not contain "xliff" or "file" node(s) required for parsing.'); + } + fileNodes.forEach(function (file) { + var originalFilePath = file.$.original; + if (!originalFilePath) { + reject('XLIFF file node does not contain original attribute to determine the original location of the resource file.'); + } + var language = file.$['target-language'].toLowerCase(); + if (!language) { + reject('XLIFF file node does not contain target-language attribute to determine translated language.'); + } + var messages = {}; + var transUnits = file.body[0]['trans-unit']; + transUnits.forEach(function (unit) { + var key = unit.$.id; + if (!unit.target) { + return; // No translation available + } + var val = unit.target.toString(); + if (key && val) { + messages[key] = decodeEntities(val); + } + else { + reject('XLIFF file does not contain full localization data. ID or target translation for one of the trans-unit nodes is not present.'); + } + }); + files.push({ messages: messages, originalFilePath: originalFilePath, language: language }); + }); + resolve(files); + }); + }); +}; +exports.XLF = XLF; + var vscodeLanguages = [ 'chs', 'cht', @@ -68,6 +243,28 @@ var iso639_3_to_2 = { 'sve': 'sv-se', 'trk': 'tr' }; + +var iso639_2_to_3 = { + 'zh-cn': 'chs', + 'zh-tw': 'cht', + 'cs-cz': 'csy', + 'de': 'deu', + 'en': 'enu', + 'es': 'esn', + 'fr': 'fra', + 'hu': 'hun', + 'it': 'ita', + 'ja': 'jpn', + 'ko': 'kor', + 'nl': 'nld', + 'pl': 'plk', + 'pt-br': 'ptb', + 'pt': 'ptg', + 'ru': 'rus', + 'sv-se': 'sve', + 'tr': 'trk' +}; + function sortLanguages(directoryNames) { return directoryNames.map(function (dirName) { var lower = dirName.toLowerCase(); @@ -288,3 +485,472 @@ function processNlsFiles(opts) { }); } exports.processNlsFiles = processNlsFiles; + +function prepareXlfFiles(projectName, extensionName) { + return event_stream_1.through(function (file) { + if (!file.isBuffer()) { + log('Error', "Failed to read component file: " + file.relative); + } + var extension = path.extname(file.path); + if (extension === '.json') { + var json = JSON.parse(file.contents.toString('utf8')); + if (BundledFormat.is(json)) { + importBundleJson(file, json, this); + } + else if (PackageJsonFormat.is(json) || ModuleJsonFormat.is(json)) { + importModuleOrPackageJson(file, json, projectName, this, extensionName); + } + else { + log('Error', 'JSON format cannot be deduced.'); + } + } + else if (extension === '.isl') { + importIsl(file, this); + } + }); +} +exports.prepareXlfFiles = prepareXlfFiles; +function getResource(sourceFile) { + var editorProject = 'vscode-editor', workbenchProject = 'vscode-workbench'; + var resource; + if (sourceFile.startsWith('vs/platform')) { + return { name: 'vs/platform', project: editorProject }; + } + else if (sourceFile.startsWith('vs/editor/contrib')) { + return { name: 'vs/editor/contrib', project: editorProject }; + } + else if (sourceFile.startsWith('vs/editor')) { + return { name: 'vs/editor', project: editorProject }; + } + else if (sourceFile.startsWith('vs/base')) { + return { name: 'vs/base', project: editorProject }; + } + else if (sourceFile.startsWith('vs/code')) { + return { name: 'vs/code', project: workbenchProject }; + } + else if (sourceFile.startsWith('vs/workbench/parts')) { + resource = sourceFile.split('/', 4).join('/'); + return { name: resource, project: workbenchProject }; + } + else if (sourceFile.startsWith('vs/workbench/services')) { + resource = sourceFile.split('/', 4).join('/'); + return { name: resource, project: workbenchProject }; + } + else if (sourceFile.startsWith('vs/workbench')) { + return { name: 'vs/workbench', project: workbenchProject }; + } + throw new Error("Could not identify the XLF bundle for " + sourceFile); +} +function importBundleJson(file, json, stream) { + var transifexEditorXlfs = Object.create(null); + for (var source in json.keys) { + var projectResource = getResource(source); + var resource = projectResource.name; + var project = projectResource.project; + var keys = json.keys[source]; + var messages = json.messages[source]; + if (keys.length !== messages.length) { + log('Error:', "There is a mismatch between keys and messages in " + file.relative); + } + var xlf = transifexEditorXlfs[resource] ? transifexEditorXlfs[resource] : transifexEditorXlfs[resource] = new XLF(project); + xlf.addFile(source, keys, messages); + } + for (var resource in transifexEditorXlfs) { + var newFilePath = transifexEditorXlfs[resource].project + "/" + resource.replace(/\//g, '_') + ".xlf"; + var xlfFile = new File({ path: newFilePath, contents: new Buffer(transifexEditorXlfs[resource].toString(), 'utf-8') }); + stream.emit('data', xlfFile); + } +} +// Keeps existing XLF instances and a state of how many files were already processed for faster file emission +var extensions = Object.create(null); +function importModuleOrPackageJson(file, json, projectName, stream, extensionName) { + if (ModuleJsonFormat.is(json) && json['keys'].length !== json['messages'].length) { + log('Error:', "There is a mismatch between keys and messages in " + file.relative); + } + // Prepare the source path for attribute in XLF & extract messages from JSON + var formattedSourcePath = file.relative.replace(/\\/g, '/'); + var messages = Object.keys(json).map(function (key) { return json[key].toString(); }); + // Stores the amount of localization files to be transformed to XLF before the emission + var localizationFilesCount, originalFilePath; + // If preparing XLF for external extension, then use different glob pattern and source path + if (extensionName) { + localizationFilesCount = glob.sync('**/*.nls.json').length; + originalFilePath = "" + formattedSourcePath.substr(0, formattedSourcePath.length - '.nls.json'.length); + } + else { + // Used for vscode/extensions folder + extensionName = formattedSourcePath.split('/')[0]; + localizationFilesCount = glob.sync("./extensions/" + extensionName + "/**/*.nls.json").length; + originalFilePath = "extensions/" + formattedSourcePath.substr(0, formattedSourcePath.length - '.nls.json'.length); + } + var extension = extensions[extensionName] ? + extensions[extensionName] : extensions[extensionName] = { xlf: new XLF(projectName), processed: 0 }; + if (ModuleJsonFormat.is(json)) { + extension.xlf.addFile(originalFilePath, json['keys'], json['messages']); + } + else { + extension.xlf.addFile(originalFilePath, Object.keys(json), messages); + } + // Check if XLF is populated with file nodes to emit it + if (++extensions[extensionName].processed === localizationFilesCount) { + var newFilePath = path.join(projectName, extensionName + '.xlf'); + var xlfFile = new File({ path: newFilePath, contents: new Buffer(extension.xlf.toString(), 'utf-8') }); + stream.emit('data', xlfFile); + } +} +var islXlf, islProcessed = 0; +function importIsl(file, stream) { + var islFiles = ['Default.isl', 'messages.en.isl']; + var projectName = 'vscode-workbench'; + var xlf = islXlf ? islXlf : islXlf = new XLF(projectName), keys = [], messages = []; + var model = new TextModel(file.contents.toString()); + var inMessageSection = false; + model.lines.forEach(function (line) { + if (line.length === 0) { + return; + } + var firstChar = line.charAt(0); + switch (firstChar) { + case ';': + // Comment line; + return; + case '[': + inMessageSection = '[Messages]' === line || '[CustomMessages]' === line; + return; + } + if (!inMessageSection) { + return; + } + var sections = line.split('='); + if (sections.length !== 2) { + log('Error:', "Badly formatted message found: " + line); + } + else { + var key = sections[0]; + var value = sections[1]; + if (key.length > 0 && value.length > 0) { + keys.push(key); + messages.push(value); + } + } + }); + var originalPath = file.path.substring(file.cwd.length + 1, file.path.split('.')[0].length).replace(/\\/g, '/'); + xlf.addFile(originalPath, keys, messages); + // Emit only upon all ISL files combined into single XLF instance + if (++islProcessed === islFiles.length) { + var newFilePath = path.join(projectName, 'setup.xlf'); + var xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8') }); + stream.emit('data', xlfFile); + } +} +function pushXlfFiles(apiUrl, username, password) { + return event_stream_1.through(function (file) { + var project = path.dirname(file.relative); + var fileName = path.basename(file.path); + var slug = fileName.substr(0, fileName.length - '.xlf'.length); + var credentials = { + 'user': username, + 'password': password + }; + // Check if resource already exists, if not, then create it. + tryGetResource(project, slug, apiUrl, credentials).then(function (exists) { + if (exists) { + updateResource(project, slug, file, apiUrl, credentials); + } + else { + createResource(project, slug, file, apiUrl, credentials); + } + }); + }); +} +exports.pushXlfFiles = pushXlfFiles; +function tryGetResource(project, slug, apiUrl, credentials) { + return new Promise(function (resolve, reject) { + var url = apiUrl + "/project/" + project + "/resource/" + slug + "/?details"; + request.get(url, { 'auth': credentials }).on('response', function (response) { + if (response.statusCode === 404) { + resolve(false); + } + else if (response.statusCode === 200) { + resolve(true); + } + else { + reject("Failed to query resource " + slug + ". Response: " + response.statusCode + " " + response.statusMessage); + } + }); + }); +} +function createResource(project, slug, xlfFile, apiUrl, credentials) { + var url = apiUrl + "/project/" + project + "/resources"; + var options = { + 'body': { + 'content': xlfFile.contents.toString(), + 'name': slug, + 'slug': slug, + 'i18n_type': 'XLIFF' + }, + 'json': true, + 'auth': credentials + }; + request.post(url, options, function (err, res) { + if (err) { + log('Error:', "Failed to create Transifex " + project + "/" + slug + ": " + err); + } + if (res.statusCode === 201) { + log("Resource " + project + "/" + slug + " successfully created on Transifex."); + } + else { + log('Error:', "Something went wrong creating " + slug + " in " + project + ". " + res.statusCode); + } + }); +} +/** + * The following link provides information about how Transifex handles updates of a resource file: + * https://dev.befoolish.co/tx-docs/public/projects/updating-content#what-happens-when-you-update-files + */ +function updateResource(project, slug, xlfFile, apiUrl, credentials) { + var url = apiUrl + "/project/" + project + "/resource/" + slug + "/content"; + var options = { + 'body': { 'content': xlfFile.contents.toString() }, + 'json': true, + 'auth': credentials + }; + request.put(url, options, function (err, res, body) { + if (err) { + log('Error:', "Failed to update Transifex " + project + "/" + slug + ": " + err); + } + if (res.statusCode === 200) { + log("Resource " + project + "/" + slug + " successfully updated on Transifex. Strings added: " + body['strings_added'] + ", updated: " + body['strings_updated'] + ", deleted: " + body['strings_delete']); + } + else { + log('Error:', "Something went wrong updating " + slug + " in " + project + ". " + res.statusCode); + } + }); +} +function getMetadataResources(pathToMetadata) { + var metadata = fs.readFileSync(pathToMetadata).toString('utf8'); + var json = JSON.parse(metadata); + var slugs = []; + var _loop_1 = function (source) { + var projectResource = getResource(source); + if (!slugs.find(function (slug) { return slug.name === projectResource.name && slug.project === projectResource.project; })) { + slugs.push(projectResource); + } + }; + for (var source in json['keys']) { + _loop_1(source); + } + return slugs; +} +function obtainProjectResources(projectName) { + var resources; + if (projectName === 'vscode-editor-workbench') { + resources = getMetadataResources('./out-vscode/nls.metadata.json'); + resources.push({ name: 'setup', project: 'vscode-workbench' }); + } + else if (projectName === 'vscode-extensions') { + var extensionsToLocalize = glob.sync('./extensions/**/*.nls.json').map(function (extension) { return extension.split('/')[2]; }); + var resourcesToPull_1 = []; + resources = []; + extensionsToLocalize.forEach(function (extension) { + if (resourcesToPull_1.indexOf(extension) === -1) { + resourcesToPull_1.push(extension); + resources.push({ name: extension, project: projectName }); + } + }); + } + return resources; +} +function pullXlfFiles(projectName, apiUrl, username, password, resources) { + if (!resources) { + resources = obtainProjectResources(projectName); + } + if (!resources) { + throw new Error('Transifex projects and resources must be defined to be able to pull translations from Transifex.'); + } + var credentials = { + 'auth': { + 'user': username, + 'password': password + } + }; + var expectedTranslationsCount = vscodeLanguages.length * resources.length; + var translationsRetrieved = 0, called = false; + return es.readable(function (count, callback) { + // Mark end of stream when all resources were retrieved + if (translationsRetrieved === expectedTranslationsCount) { + return this.emit('end'); + } + if (!called) { + called = true; + var stream_1 = this; + vscodeLanguages.map(function (language) { + resources.map(function (resource) { + var slug = resource.name.replace(/\//g, '_'); + var project = resource.project; + var iso639 = iso639_3_to_2[language]; + var url = apiUrl + "/project/" + project + "/resource/" + slug + "/translation/" + iso639 + "?file&mode=onlyreviewed"; + var xlfBuffer = '', responseCode; + request.get(url, credentials) + .on('response', function (response) { + responseCode = response.statusCode; + }) + .on('data', function (data) { return xlfBuffer += data; }) + .on('end', function () { + if (responseCode === 200) { + stream_1.emit('data', new File({ contents: new Buffer(xlfBuffer) })); + } + else { + log('Error:', slug + " in " + project + " returned no data. Response code: " + responseCode + "."); + } + translationsRetrieved++; + }) + .on('error', function (error) { + log('Error:', "Failed to query resource " + slug + " with the following error: " + error); + }); + }); + }); + } + callback(); + }); +} +exports.pullXlfFiles = pullXlfFiles; +function prepareJsonFiles() { + return event_stream_1.through(function (xlf) { + var stream = this; + XLF.parse(xlf.contents.toString()).then(function (resolvedFiles) { + resolvedFiles.forEach(function (file) { + var messages = file.messages, translatedFile; + // ISL file path always starts with 'build/' + if (file.originalFilePath.startsWith('build/')) { + var defaultLanguages = { 'zh-cn': true, 'zh-tw': true, 'ko': true }; + if (path.basename(file.originalFilePath) === 'Default' && !defaultLanguages[file.language]) { + return; + } + translatedFile = createIslFile('..', file.originalFilePath, messages, iso639_2_to_3[file.language]); + } + else { + translatedFile = createI18nFile(iso639_2_to_3[file.language], file.originalFilePath, messages); + } + stream.emit('data', translatedFile); + }); + }, function (rejectReason) { + log('Error:', rejectReason); + }); + }); +} +exports.prepareJsonFiles = prepareJsonFiles; +function createI18nFile(base, originalFilePath, messages) { + var content = [ + '/*---------------------------------------------------------------------------------------------', + ' * Copyright (c) Microsoft Corporation. All rights reserved.', + ' * Licensed under the MIT License. See License.txt in the project root for license information.', + ' *--------------------------------------------------------------------------------------------*/', + '// Do not edit this file. It is machine generated.' + ].join('\n') + '\n' + JSON.stringify(messages, null, '\t').replace(/\r\n/g, '\n'); + return new File({ + path: path.join(base, originalFilePath + '.i18n.json'), + contents: new Buffer(content, 'utf8') + }); +} +exports.createI18nFile = createI18nFile; +var languageNames = { + 'chs': 'Simplified Chinese', + 'cht': 'Traditional Chinese', + 'kor': 'Korean' +}; +var languageIds = { + 'chs': '$0804', + 'cht': '$0404', + 'kor': '$0412' +}; +var encodings = { + 'chs': 'CP936', + 'cht': 'CP950', + 'jpn': 'CP932', + 'kor': 'CP949', + 'deu': 'CP1252', + 'fra': 'CP1252', + 'esn': 'CP1252', + 'rus': 'CP1251', + 'ita': 'CP1252' +}; +function createIslFile(base, originalFilePath, messages, language) { + var content = []; + var originalContent; + if (path.basename(originalFilePath) === 'Default') { + originalContent = new TextModel(fs.readFileSync(originalFilePath + '.isl', 'utf8')); + } + else { + originalContent = new TextModel(fs.readFileSync(originalFilePath + '.en.isl', 'utf8')); + } + originalContent.lines.forEach(function (line) { + if (line.length > 0) { + var firstChar = line.charAt(0); + if (firstChar === '[' || firstChar === ';') { + if (line === '; *** Inno Setup version 5.5.3+ English messages ***') { + content.push("; *** Inno Setup version 5.5.3+ " + languageNames[language] + " messages ***"); + } + else { + content.push(line); + } + } + else { + var sections = line.split('='); + var key = sections[0]; + var translated = line; + if (key) { + if (key === 'LanguageName') { + translated = key + "=" + languageNames[language]; + } + else if (key === 'LanguageID') { + translated = key + "=" + languageIds[language]; + } + else if (key === 'LanguageCodePage') { + translated = key + "=" + encodings[language].substr(2); + } + else { + var translatedMessage = messages[key]; + if (translatedMessage) { + translated = key + "=" + translatedMessage; + } + } + } + content.push(translated); + } + } + }); + var tag = iso639_3_to_2[language]; + var basename = path.basename(originalFilePath); + var filePath = path.join(base, path.dirname(originalFilePath), basename) + "." + tag + ".isl"; + return new File({ + path: filePath, + contents: iconv.encode(new Buffer(content.join('\r\n'), 'utf8'), encodings[language]) + }); +} +exports.createIslFile = createIslFile; +function encodeEntities(value) { + var result = []; + for (var i = 0; i < value.length; i++) { + var ch = value[i]; + switch (ch) { + case '<': + result.push('<'); + break; + case '>': + result.push('>'); + break; + case '&': + result.push('&'); + break; + default: + result.push(ch); + } + } + return result.join(''); +} +function decodeEntities(value) { + return value.replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&'); +} +exports.decodeEntities = decodeEntities; +//# sourceMappingURL=i18n.js.map diff --git a/build/lib/i18n.ts b/build/lib/i18n.ts index 046cd8c8d8b..146664b6a34 100644 --- a/build/lib/i18n.ts +++ b/build/lib/i18n.ts @@ -10,8 +10,13 @@ import { through } from 'event-stream'; import { ThroughStream } from 'through'; import File = require('vinyl'); import * as Is from 'is'; +import xml2js = require('xml2js'); +import * as glob from 'glob'; var util = require('gulp-util'); +const request = require('request'); +const es = require('event-stream'); +var iconv = require('iconv-lite'); function log(message: any, ...rest: any[]): void { util.log(util.colors.green('[i18n]'), message, ...rest); @@ -21,6 +26,17 @@ interface Map { [key: string]: V; } +interface Item { + id: string; + message: string; + comment: string; +} + +interface Resource { + name: string; + project: string; +} + interface LocalizeInfo { key: string; comment: string[]; @@ -52,6 +68,205 @@ module BundledFormat { } } +interface ValueFormat { + message: string; + comment: string[]; +} + +interface PackageJsonFormat { + [key: string]: string | ValueFormat; +} + +module PackageJsonFormat { + export function is(value: any): value is PackageJsonFormat { + if (Is.undef(value) || !Is.object(value)) { + return false; + } + return Object.keys(value).every(key => { + let element = value[key]; + return Is.string(element) || (Is.object(element) && Is.defined(element.message) && Is.defined(element.comment)); + }); + } +} + +interface ModuleJsonFormat { + messages: string[]; + keys: (string | LocalizeInfo)[]; +} + +module ModuleJsonFormat { + export function is(value: any): value is ModuleJsonFormat { + let candidate = value as ModuleJsonFormat; + return Is.defined(candidate) + && Is.array(candidate.messages) && candidate.messages.every(message => Is.string(message)) + && Is.array(candidate.keys) && candidate.keys.every(key => Is.string(key) || LocalizeInfo.is(key)); + } +} + +export class Line { + private buffer: string[] = []; + + constructor(private indent: number = 0) { + if (indent > 0) { + this.buffer.push(new Array(indent + 1).join(' ')); + } + } + + public append(value: string): Line { + this.buffer.push(value); + return this; + } + + public toString(): string { + return this.buffer.join(''); + } +} + +class TextModel { + private _lines: string[]; + + constructor(contents: string) { + this._lines = contents.split(/\r\n|\r|\n/); + } + + public get lines(): string[] { + return this._lines; + } +} + +export class XLF { + private buffer: string[]; + private files: Map; + + constructor(public project: string) { + this.buffer = []; + this.files = Object.create(null); + } + + public toString(): string { + this.appendHeader(); + + for (let file in this.files) { + this.appendNewLine(``, 2); + for (let item of this.files[file]) { + this.addStringItem(item); + } + this.appendNewLine('', 2); + } + + this.appendFooter(); + return this.buffer.join('\r\n'); + } + + public addFile(original: string, keys: any[], messages: string[]) { + this.files[original] = []; + let existingKeys = []; + + for (let key of keys) { + // Ignore duplicate keys because Transifex does not populate those with translated values. + if (existingKeys.indexOf(key) !== -1) { + continue; + } + existingKeys.push(key); + + let message: string = encodeEntities(messages[keys.indexOf(key)]); + let comment: string = undefined; + + // Check if the message contains description (if so, it becomes an object type in JSON) + if (Is.string(key)) { + this.files[original].push({ id: key, message: message, comment: comment }); + } else { + if (key['comment'] && key['comment'].length > 0) { + comment = key['comment'].map(comment => encodeEntities(comment)).join('\r\n'); + } + + this.files[original].push({ id: key['key'], message: message, comment: comment }); + } + } + } + + private addStringItem(item: Item): void { + if (!item.id || !item.message) { + //throw new Error('No item ID or value specified.'); + } + + this.appendNewLine(``, 4); + this.appendNewLine(`${encodeEntities(item.message)}`, 6); + + if (item.comment) { + this.appendNewLine(`${item.comment}`, 6); + } + + this.appendNewLine('', 4); + } + + private appendHeader(): void { + this.appendNewLine('', 0); + this.appendNewLine('', 0); + } + + private appendFooter(): void { + this.appendNewLine('', 0); + } + + private appendNewLine(content: string, indent?: number): void { + let line = new Line(indent); + line.append(content); + this.buffer.push(line.toString()); + } + + static parse = function(xlfString: string) : Promise<{ messages: Map, originalFilePath: string, language: string }[]> { + return new Promise((resolve, reject) => { + let parser = new xml2js.Parser(); + + let files: { messages: Map, originalFilePath: string, language: string }[] = []; + + parser.parseString(xlfString, function(err, result) { + if (err) { + reject(`Failed to parse XLIFF string. ${err}`); + } + + const fileNodes: any[] = result['xliff']['file']; + if (!fileNodes) { + reject('XLIFF file does not contain "xliff" or "file" node(s) required for parsing.'); + } + + fileNodes.forEach((file) => { + const originalFilePath = file.$.original; + if (!originalFilePath) { + reject('XLIFF file node does not contain original attribute to determine the original location of the resource file.'); + } + const language = file.$['target-language'].toLowerCase(); + if (!language) { + reject('XLIFF file node does not contain target-language attribute to determine translated language.'); + } + + let messages: Map = {}; + const transUnits = file.body[0]['trans-unit']; + + transUnits.forEach(unit => { + const key = unit.$.id; + if (!unit.target) { + return; // No translation available + } + + const val = unit.target.toString(); + if (key && val) { + messages[key] = decodeEntities(val); + } else { + reject('XLIFF file does not contain full localization data. ID or target translation for one of the trans-unit nodes is not present.'); + } + }); + + files.push({ messages: messages, originalFilePath: originalFilePath, language: language }); + }); + + resolve(files); + }); + }); + }; +} + const vscodeLanguages: string[] = [ 'chs', 'cht', @@ -85,6 +300,26 @@ const iso639_3_to_2: Map = { 'trk': 'tr' }; +const iso639_2_to_3: Map = { + 'zh-cn': 'chs', + 'zh-tw': 'cht', + 'cs-cz': 'csy', + 'de': 'deu', + 'en': 'enu', + 'es': 'esn', + 'fr': 'fra', + 'hu': 'hun', + 'it': 'ita', + 'ja': 'jpn', + 'ko': 'kor', + 'nl': 'nld', + 'pl': 'plk', + 'pt-br': 'ptb', + 'pt': 'ptg', + 'ru': 'rus', + 'sv-se': 'sve', + 'tr': 'trk' +}; interface IDirectoryInfo { name: string; iso639_2: string; @@ -138,7 +373,7 @@ function stripComments(content: string): string { } }); return result; -}; +} function escapeCharacters(value:string):string { var result:string[] = []; @@ -308,4 +543,509 @@ export function processNlsFiles(opts:{fileHeader:string;}): ThroughStream { } this.emit('data', file); }); +} + +export function prepareXlfFiles(projectName?: string, extensionName?: string): ThroughStream { + return through( + function (file: File) { + if (!file.isBuffer()) { + log('Error', `Failed to read component file: ${file.relative}`); + } + + const extension = path.extname(file.path); + if (extension === '.json') { + const json = JSON.parse((file.contents).toString('utf8')); + + if (BundledFormat.is(json)) { + importBundleJson(file, json, this); + } else if (PackageJsonFormat.is(json) || ModuleJsonFormat.is(json)) { + importModuleOrPackageJson(file, json, projectName, this, extensionName); + } else { + log('Error', 'JSON format cannot be deduced.'); + } + } else if (extension === '.isl') { + importIsl(file, this); + } + } + ); +} + +function getResource(sourceFile: string): Resource { + const editorProject: string = 'vscode-editor', + workbenchProject: string = 'vscode-workbench'; + let resource: string; + + if (sourceFile.startsWith('vs/platform')) { + return { name: 'vs/platform', project: editorProject }; + } else if (sourceFile.startsWith('vs/editor/contrib')) { + return { name: 'vs/editor/contrib', project: editorProject }; + } else if (sourceFile.startsWith('vs/editor')) { + return { name: 'vs/editor', project: editorProject }; + } else if (sourceFile.startsWith('vs/base')) { + return { name: 'vs/base', project: editorProject }; + } else if (sourceFile.startsWith('vs/code')) { + return { name: 'vs/code', project: workbenchProject }; + } else if (sourceFile.startsWith('vs/workbench/parts')) { + resource = sourceFile.split('/', 4).join('/'); + return { name: resource, project: workbenchProject }; + } else if (sourceFile.startsWith('vs/workbench/services')) { + resource = sourceFile.split('/', 4).join('/'); + return { name: resource, project: workbenchProject }; + } else if (sourceFile.startsWith('vs/workbench')) { + return { name: 'vs/workbench', project: workbenchProject }; + } + + throw new Error (`Could not identify the XLF bundle for ${sourceFile}`); +} + + +function importBundleJson(file: File, json: BundledFormat, stream: ThroughStream): void { + let transifexEditorXlfs: Map = Object.create(null); + + for (let source in json.keys) { + const projectResource = getResource(source); + const resource = projectResource.name; + const project = projectResource.project; + + const keys = json.keys[source]; + const messages = json.messages[source]; + if (keys.length !== messages.length) { + log('Error:', `There is a mismatch between keys and messages in ${file.relative}`); + } + + let xlf = transifexEditorXlfs[resource] ? transifexEditorXlfs[resource] : transifexEditorXlfs[resource] = new XLF(project); + xlf.addFile(source, keys, messages); + } + + for (let resource in transifexEditorXlfs) { + const newFilePath = `${transifexEditorXlfs[resource].project}/${resource.replace(/\//g, '_')}.xlf`; + const xlfFile = new File({ path: newFilePath, contents: new Buffer(transifexEditorXlfs[resource].toString(), 'utf-8')}); + stream.emit('data', xlfFile); + } +} + +// Keeps existing XLF instances and a state of how many files were already processed for faster file emission +var extensions: Map<{ xlf: XLF, processed: number }> = Object.create(null); +function importModuleOrPackageJson(file: File, json: ModuleJsonFormat | PackageJsonFormat, projectName: string, stream: ThroughStream, extensionName?: string): void { + if (ModuleJsonFormat.is(json) && json['keys'].length !== json['messages'].length) { + log('Error:', `There is a mismatch between keys and messages in ${file.relative}`); + } + + // Prepare the source path for attribute in XLF & extract messages from JSON + const formattedSourcePath = file.relative.replace(/\\/g, '/'); + const messages = Object.keys(json).map((key) => json[key].toString()); + + // Stores the amount of localization files to be transformed to XLF before the emission + let localizationFilesCount, + originalFilePath; + // If preparing XLF for external extension, then use different glob pattern and source path + if (extensionName) { + localizationFilesCount = glob.sync('**/*.nls.json').length; + originalFilePath = `${formattedSourcePath.substr(0, formattedSourcePath.length - '.nls.json'.length)}`; + } else { + // Used for vscode/extensions folder + extensionName = formattedSourcePath.split('/')[0]; + localizationFilesCount = glob.sync(`./extensions/${extensionName}/**/*.nls.json`).length; + originalFilePath = `extensions/${formattedSourcePath.substr(0, formattedSourcePath.length - '.nls.json'.length)}`; + } + + let extension = extensions[extensionName] ? + extensions[extensionName] : extensions[extensionName] = { xlf: new XLF(projectName), processed: 0 }; + + if (ModuleJsonFormat.is(json)) { + extension.xlf.addFile(originalFilePath, json['keys'], json['messages']); + } else { + extension.xlf.addFile(originalFilePath, Object.keys(json), messages); + } + + // Check if XLF is populated with file nodes to emit it + if (++extensions[extensionName].processed === localizationFilesCount) { + const newFilePath = path.join(projectName, extensionName + '.xlf'); + const xlfFile = new File({ path: newFilePath, contents: new Buffer(extension.xlf.toString(), 'utf-8')}); + stream.emit('data', xlfFile); + } +} + +var islXlf: XLF, + islProcessed: number = 0; + +function importIsl(file: File, stream: ThroughStream) { + const islFiles = ['Default.isl', 'messages.en.isl']; + const projectName = 'vscode-workbench'; + + let xlf = islXlf ? islXlf : islXlf = new XLF(projectName), + keys: string[] = [], + messages: string[] = []; + + let model = new TextModel(file.contents.toString()); + let inMessageSection = false; + model.lines.forEach(line => { + if (line.length === 0) { + return; + } + let firstChar = line.charAt(0); + switch (firstChar) { + case ';': + // Comment line; + return; + case '[': + inMessageSection = '[Messages]' === line || '[CustomMessages]' === line; + return; + } + if (!inMessageSection) { + return; + } + let sections: string[] = line.split('='); + if (sections.length !== 2) { + log('Error:', `Badly formatted message found: ${line}`); + } else { + let key = sections[0]; + let value = sections[1]; + if (key.length > 0 && value.length > 0) { + keys.push(key); + messages.push(value); + } + } + }); + + const originalPath = file.path.substring(file.cwd.length+1, file.path.split('.')[0].length).replace(/\\/g, '/'); + xlf.addFile(originalPath, keys, messages); + + // Emit only upon all ISL files combined into single XLF instance + if (++islProcessed === islFiles.length) { + const newFilePath = path.join(projectName, 'setup.xlf'); + const xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8')}); + stream.emit('data', xlfFile); + } +} + +export function pushXlfFiles(apiUrl: string, username: string, password: string): ThroughStream { + return through(function(file: File) { + const project = path.dirname(file.relative); + const fileName = path.basename(file.path); + const slug = fileName.substr(0, fileName.length - '.xlf'.length); + const credentials = { + 'user': username, + 'password': password + }; + + // Check if resource already exists, if not, then create it. + tryGetResource(project, slug, apiUrl, credentials).then(exists => { + if (exists) { + updateResource(project, slug, file, apiUrl, credentials); + } else { + createResource(project, slug, file, apiUrl, credentials); + } + }); + }); +} + +function tryGetResource(project: string, slug: string, apiUrl: string, credentials: any): Promise { + return new Promise((resolve, reject) => { + const url = `${apiUrl}/project/${project}/resource/${slug}/?details`; + request.get(url, { 'auth': credentials }).on('response', function (response) { + if (response.statusCode === 404) { + resolve(false); + } else if (response.statusCode === 200) { + resolve(true); + } else { + reject(`Failed to query resource ${slug}. Response: ${response.statusCode} ${response.statusMessage}`); + } + }); + }); +} + +function createResource(project: string, slug: string, xlfFile: File, apiUrl:string, credentials: any): void { + const url = `${apiUrl}/project/${project}/resources`; + const options = { + 'body': { + 'content': xlfFile.contents.toString(), + 'name': slug, + 'slug': slug, + 'i18n_type': 'XLIFF' + }, + 'json': true, + 'auth': credentials + }; + + request.post(url, options, function(err, res) { + if (err) { + log('Error:', `Failed to create Transifex ${project}/${slug}: ${err}`); + } + + if (res.statusCode === 201) { + log(`Resource ${project}/${slug} successfully created on Transifex.`); + } else { + log('Error:', `Something went wrong creating ${slug} in ${project}. ${res.statusCode}`); + } + }); +} + +/** + * The following link provides information about how Transifex handles updates of a resource file: + * https://dev.befoolish.co/tx-docs/public/projects/updating-content#what-happens-when-you-update-files + */ +function updateResource(project: string, slug: string, xlfFile: File, apiUrl: string, credentials: any) : void { + const url = `${apiUrl}/project/${project}/resource/${slug}/content`; + const options = { + 'body': { 'content': xlfFile.contents.toString() }, + 'json': true, + 'auth': credentials + }; + + request.put(url, options, function(err, res, body) { + if (err) { + log('Error:', `Failed to update Transifex ${project}/${slug}: ${err}`); + } + + if (res.statusCode === 200) { + log(`Resource ${project}/${slug} successfully updated on Transifex. Strings added: ${body['strings_added']}, updated: ${body['strings_updated']}, deleted: ${body['strings_delete']}`); + } else { + log('Error:', `Something went wrong updating ${slug} in ${project}. ${res.statusCode}`); + } + }); +} + +function getMetadataResources(pathToMetadata: string) : Resource[] { + const metadata = fs.readFileSync(pathToMetadata).toString('utf8'); + const json = JSON.parse(metadata); + let slugs = []; + + for (let source in json['keys']) { + let projectResource = getResource(source); + if (!slugs.find(slug => slug.name === projectResource.name && slug.project === projectResource.project)) { + slugs.push(projectResource); + } + } + + return slugs; +} + +function obtainProjectResources(projectName: string): Resource[] { + let resources: Resource[]; + + if (projectName === 'vscode-editor-workbench') { + resources = getMetadataResources('./out-vscode/nls.metadata.json'); + resources.push({ name: 'setup', project: 'vscode-workbench' }); + } else if (projectName === 'vscode-extensions') { + let extensionsToLocalize: string[] = glob.sync('./extensions/**/*.nls.json').map(extension => extension.split('/')[2]); + let resourcesToPull: string[] = []; + resources = []; + + extensionsToLocalize.forEach(extension => { + if (resourcesToPull.indexOf(extension) === -1) { // remove duplicate elements returned by glob + resourcesToPull.push(extension); + resources.push({ name: extension, project: projectName }); + } + }); + } + + return resources; +} + +export function pullXlfFiles(projectName: string, apiUrl: string, username: string, password: string, resources?: Resource[]): NodeJS.ReadableStream { + if (!resources) { + resources = obtainProjectResources(projectName); + } + if (!resources) { + throw new Error('Transifex projects and resources must be defined to be able to pull translations from Transifex.'); + } + + const credentials = { + 'auth': { + 'user': username, + 'password': password + } + }; + let expectedTranslationsCount = vscodeLanguages.length * resources.length; + let translationsRetrieved = 0, called = false; + + return es.readable(function(count, callback) { + // Mark end of stream when all resources were retrieved + if (translationsRetrieved === expectedTranslationsCount) { + return this.emit('end'); + } + + if (!called) { + called = true; + const stream = this; + + vscodeLanguages.map(function(language) { + resources.map(function (resource) { + const slug = resource.name.replace(/\//g, '_'); + const project = resource.project; + const iso639 = iso639_3_to_2[language]; + const url = `${apiUrl}/project/${project}/resource/${slug}/translation/${iso639}?file&mode=onlyreviewed`; + + let xlfBuffer: string = '', responseCode: number; + request.get(url, credentials) + .on('response', (response) => { + responseCode = response.statusCode; + }) + .on('data', (data) => xlfBuffer += data) + .on('end', () => { + if (responseCode === 200) { + stream.emit('data', new File({ contents: new Buffer(xlfBuffer) })); + } else { + log('Error:', `${slug} in ${project} returned no data. Response code: ${responseCode}.`); + } + translationsRetrieved++; + }) + .on('error', (error) => { + log('Error:', `Failed to query resource ${slug} with the following error: ${error}`); + }); + }); + }); + } + + callback(); + }); +} + +export function prepareJsonFiles(): ThroughStream { + return through(function(xlf: File) { + let stream = this; + + XLF.parse(xlf.contents.toString()).then( + function(resolvedFiles) { + resolvedFiles.forEach(file => { + let messages = file.messages, translatedFile; + + // ISL file path always starts with 'build/' + if (file.originalFilePath.startsWith('build/')) { + const defaultLanguages = { 'zh-cn': true, 'zh-tw': true, 'ko': true }; + if (path.basename(file.originalFilePath) === 'Default' && !defaultLanguages[file.language]) { + return; + } + + translatedFile = createIslFile('..', file.originalFilePath, messages, iso639_2_to_3[file.language]); + } else { + translatedFile = createI18nFile(iso639_2_to_3[file.language], file.originalFilePath, messages); + } + + stream.emit('data', translatedFile); + }); + }, + function(rejectReason) { + log('Error:', rejectReason); + } + ); + }); +} + +export function createI18nFile(base: string, originalFilePath: string, messages: Map): File { + let content = [ + '/*---------------------------------------------------------------------------------------------', + ' * Copyright (c) Microsoft Corporation. All rights reserved.', + ' * Licensed under the MIT License. See License.txt in the project root for license information.', + ' *--------------------------------------------------------------------------------------------*/', + '// Do not edit this file. It is machine generated.' + ].join('\n') + '\n' + JSON.stringify(messages, null, '\t').replace(/\r\n/g, '\n'); + + return new File({ + path: path.join(base, originalFilePath + '.i18n.json'), + contents: new Buffer(content, 'utf8') + }); +} + + +const languageNames: Map = { + 'chs': 'Simplified Chinese', + 'cht': 'Traditional Chinese', + 'kor': 'Korean' +}; + +const languageIds: Map = { + 'chs': '$0804', + 'cht': '$0404', + 'kor': '$0412' +}; + +const encodings: Map = { + 'chs': 'CP936', + 'cht': 'CP950', + 'jpn': 'CP932', + 'kor': 'CP949', + 'deu': 'CP1252', + 'fra': 'CP1252', + 'esn': 'CP1252', + 'rus': 'CP1251', + 'ita': 'CP1252' +}; + +export function createIslFile(base: string, originalFilePath: string, messages: Map, language: string): File { + let content: string[] = []; + let originalContent: TextModel; + if (path.basename(originalFilePath) === 'Default') { + originalContent = new TextModel(fs.readFileSync(originalFilePath + '.isl', 'utf8')); + } else { + originalContent = new TextModel(fs.readFileSync(originalFilePath + '.en.isl', 'utf8')); + } + + originalContent.lines.forEach(line => { + if (line.length > 0) { + let firstChar = line.charAt(0); + if (firstChar === '[' || firstChar === ';') { + if (line === '; *** Inno Setup version 5.5.3+ English messages ***') { + content.push(`; *** Inno Setup version 5.5.3+ ${languageNames[language]} messages ***`); + } else { + content.push(line); + } + } else { + let sections: string[] = line.split('='); + let key = sections[0]; + let translated = line; + if (key) { + if (key === 'LanguageName') { + translated = `${key}=${languageNames[language]}`; + } else if (key === 'LanguageID') { + translated = `${key}=${languageIds[language]}`; + } else if (key === 'LanguageCodePage') { + translated = `${key}=${encodings[language].substr(2)}`; + } else { + let translatedMessage = messages[key]; + if (translatedMessage) { + translated = `${key}=${translatedMessage}`; + } + } + } + + content.push(translated); + } + } + }); + + let tag = iso639_3_to_2[language]; + let basename = path.basename(originalFilePath); + let filePath = `${path.join(base, path.dirname(originalFilePath), basename)}.${tag}.isl`; + + return new File({ + path: filePath, + contents: iconv.encode(new Buffer(content.join('\r\n'), 'utf8'), encodings[language]) + }); +} + +function encodeEntities(value: string): string { + var result: string[] = []; + for (var i = 0; i < value.length; i++) { + var ch = value[i]; + switch (ch) { + case '<': + result.push('<'); + break; + case '>': + result.push('>'); + break; + case '&': + result.push('&'); + break; + default: + result.push(ch); + } + } + return result.join(''); +} + +export function decodeEntities(value:string): string { + return value.replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&'); } \ No newline at end of file diff --git a/extensions/typescript/package.json b/extensions/typescript/package.json index c5721ef1c0c..d93d295c725 100644 --- a/extensions/typescript/package.json +++ b/extensions/typescript/package.json @@ -403,4 +403,4 @@ } ] } -} \ No newline at end of file +} From 4fd13f7cc9b49f0dc20da1405aab456c8590c4da Mon Sep 17 00:00:00 2001 From: Michel Kaporin Date: Thu, 2 Mar 2017 21:53:02 +0100 Subject: [PATCH 2/5] Implemented transifex push and pull for translations together with json->xlf->json parsing. --- build/gulpfile.vscode.js | 33 ++ build/lib/i18n.js | 666 ++++++++++++++++++++++++++ build/lib/i18n.ts | 742 ++++++++++++++++++++++++++++- extensions/typescript/package.json | 2 +- 4 files changed, 1441 insertions(+), 2 deletions(-) diff --git a/build/gulpfile.vscode.js b/build/gulpfile.vscode.js index 74759c0235c..21432d1a217 100644 --- a/build/gulpfile.vscode.js +++ b/build/gulpfile.vscode.js @@ -29,6 +29,7 @@ const packageJson = require('../package.json'); const product = require('../product.json'); const shrinkwrap = require('../npm-shrinkwrap.json'); const crypto = require('crypto'); +const i18n = require('./lib/i18n'); const dependencies = Object.keys(shrinkwrap.dependencies) .concat(Array.isArray(product.extraNodeModules) ? product.extraNodeModules : []); // additional dependencies from our product configuration @@ -339,6 +340,38 @@ gulp.task('vscode-linux-ia32-min', ['minify-vscode', 'clean-vscode-linux-ia32'], gulp.task('vscode-linux-x64-min', ['minify-vscode', 'clean-vscode-linux-x64'], packageTask('linux', 'x64', { minified: true })); gulp.task('vscode-linux-arm-min', ['minify-vscode', 'clean-vscode-linux-arm'], packageTask('linux', 'arm', { minified: true })); +const apiUrl = process.env.TRANSIFEX_API_URL; +const apiName = process.env.TRANSIFEX_API_NAME; +const apiToken = process.env.TRANSIFEX_API_TOKEN; + +gulp.task('vscode-translations-update', function() { + const pathToMetadata = './out-vscode/nls.metadata.json'; + const pathToExtensions = './extensions/**/*.nls.json'; + const pathToSetup = 'build/win32/**/{Default.isl,messages.en.isl}'; + + gulp.src(pathToMetadata) + .pipe(i18n.prepareXlfFiles()) + .pipe(i18n.pushXlfFiles(apiUrl, apiName, apiToken)); + + gulp.src(pathToSetup) + .pipe(i18n.prepareXlfFiles()) + .pipe(i18n.pushXlfFiles(apiUrl, apiName, apiToken)); + + return gulp.src(pathToExtensions) + .pipe(i18n.prepareXlfFiles('vscode-extensions')) + .pipe(i18n.pushXlfFiles(apiUrl, apiName, apiToken)); +}); + +gulp.task('vscode-translations-pull', function() { + i18n.pullXlfFiles('vscode-editor-workbench', apiUrl, apiName, apiToken) + .pipe(i18n.prepareJsonFiles()) + .pipe(vfs.dest('./i18n')); + + return i18n.pullXlfFiles('vscode-extensions', apiUrl, apiName, apiToken) + .pipe(i18n.prepareJsonFiles()) + .pipe(vfs.dest('./i18n')); +}); + // Sourcemaps gulp.task('upload-vscode-sourcemaps', ['minify-vscode'], () => { diff --git a/build/lib/i18n.js b/build/lib/i18n.js index 7dcf0a0b763..a8256d1ad37 100644 --- a/build/lib/i18n.js +++ b/build/lib/i18n.js @@ -10,6 +10,13 @@ var event_stream_1 = require("event-stream"); var File = require("vinyl"); var Is = require("is"); var util = require('gulp-util'); +var xml2js = require("xml2js"); +var glob = require("glob"); +var util = require('gulp-util'); +var request = require('request'); +var es = require('event-stream'); +var iconv = require('iconv-lite'); + function log(message) { var rest = []; for (var _i = 1; _i < arguments.length; _i++) { @@ -37,6 +44,174 @@ var BundledFormat; } BundledFormat.is = is; })(BundledFormat || (BundledFormat = {})); + +var PackageJsonFormat; +(function (PackageJsonFormat) { + function is(value) { + if (Is.undef(value) || !Is.object(value)) { + return false; + } + return Object.keys(value).every(function (key) { + var element = value[key]; + return Is.string(element) || (Is.object(element) && Is.defined(element.message) && Is.defined(element.comment)); + }); + } + PackageJsonFormat.is = is; +})(PackageJsonFormat || (PackageJsonFormat = {})); +var ModuleJsonFormat; +(function (ModuleJsonFormat) { + function is(value) { + var candidate = value; + return Is.defined(candidate) + && Is.array(candidate.messages) && candidate.messages.every(function (message) { return Is.string(message); }) + && Is.array(candidate.keys) && candidate.keys.every(function (key) { return Is.string(key) || LocalizeInfo.is(key); }); + } + ModuleJsonFormat.is = is; +})(ModuleJsonFormat || (ModuleJsonFormat = {})); +var Line = (function () { + function Line(indent) { + if (indent === void 0) { indent = 0; } + this.indent = indent; + this.buffer = []; + if (indent > 0) { + this.buffer.push(new Array(indent + 1).join(' ')); + } + } + Line.prototype.append = function (value) { + this.buffer.push(value); + return this; + }; + Line.prototype.toString = function () { + return this.buffer.join(''); + }; + return Line; +}()); +exports.Line = Line; +var TextModel = (function () { + function TextModel(contents) { + this._lines = contents.split(/\r\n|\r|\n/); + } + Object.defineProperty(TextModel.prototype, "lines", { + get: function () { + return this._lines; + }, + enumerable: true, + configurable: true + }); + return TextModel; +}()); +var XLF = (function () { + function XLF(project) { + this.project = project; + this.buffer = []; + this.files = Object.create(null); + } + XLF.prototype.toString = function () { + this.appendHeader(); + for (var file in this.files) { + this.appendNewLine("", 2); + for (var _i = 0, _a = this.files[file]; _i < _a.length; _i++) { + var item = _a[_i]; + this.addStringItem(item); + } + this.appendNewLine('', 2); + } + this.appendFooter(); + return this.buffer.join('\r\n'); + }; + XLF.prototype.addFile = function (original, keys, messages) { + this.files[original] = []; + var existingKeys = []; + for (var _i = 0, keys_1 = keys; _i < keys_1.length; _i++) { + var key = keys_1[_i]; + // Ignore duplicate keys because Transifex does not populate those with translated values. + if (existingKeys.indexOf(key) !== -1) { + continue; + } + existingKeys.push(key); + var message = encodeEntities(messages[keys.indexOf(key)]); + var comment = undefined; + // Check if the message contains description (if so, it becomes an object type in JSON) + if (Is.string(key)) { + this.files[original].push({ id: key, message: message, comment: comment }); + } + else { + if (key['comment'] && key['comment'].length > 0) { + comment = key['comment'].map(function (comment) { return encodeEntities(comment); }).join('\r\n'); + } + this.files[original].push({ id: key['key'], message: message, comment: comment }); + } + } + }; + XLF.prototype.addStringItem = function (item) { + if (!item.id || !item.message) { + //throw new Error('No item ID or value specified.'); + } + this.appendNewLine("", 4); + this.appendNewLine("" + encodeEntities(item.message) + "", 6); + if (item.comment) { + this.appendNewLine("" + item.comment + "", 6); + } + this.appendNewLine('', 4); + }; + XLF.prototype.appendHeader = function () { + this.appendNewLine('', 0); + this.appendNewLine('', 0); + }; + XLF.prototype.appendFooter = function () { + this.appendNewLine('', 0); + }; + XLF.prototype.appendNewLine = function (content, indent) { + var line = new Line(indent); + line.append(content); + this.buffer.push(line.toString()); + }; + return XLF; +}()); +XLF.parse = function (xlfString) { + return new Promise(function (resolve, reject) { + var parser = new xml2js.Parser(); + var files = []; + parser.parseString(xlfString, function (err, result) { + if (err) { + reject("Failed to parse XLIFF string. " + err); + } + var fileNodes = result['xliff']['file']; + if (!fileNodes) { + reject('XLIFF file does not contain "xliff" or "file" node(s) required for parsing.'); + } + fileNodes.forEach(function (file) { + var originalFilePath = file.$.original; + if (!originalFilePath) { + reject('XLIFF file node does not contain original attribute to determine the original location of the resource file.'); + } + var language = file.$['target-language'].toLowerCase(); + if (!language) { + reject('XLIFF file node does not contain target-language attribute to determine translated language.'); + } + var messages = {}; + var transUnits = file.body[0]['trans-unit']; + transUnits.forEach(function (unit) { + var key = unit.$.id; + if (!unit.target) { + return; // No translation available + } + var val = unit.target.toString(); + if (key && val) { + messages[key] = decodeEntities(val); + } + else { + reject('XLIFF file does not contain full localization data. ID or target translation for one of the trans-unit nodes is not present.'); + } + }); + files.push({ messages: messages, originalFilePath: originalFilePath, language: language }); + }); + resolve(files); + }); + }); +}; +exports.XLF = XLF; + var vscodeLanguages = [ 'chs', 'cht', @@ -68,6 +243,28 @@ var iso639_3_to_2 = { 'sve': 'sv-se', 'trk': 'tr' }; + +var iso639_2_to_3 = { + 'zh-cn': 'chs', + 'zh-tw': 'cht', + 'cs-cz': 'csy', + 'de': 'deu', + 'en': 'enu', + 'es': 'esn', + 'fr': 'fra', + 'hu': 'hun', + 'it': 'ita', + 'ja': 'jpn', + 'ko': 'kor', + 'nl': 'nld', + 'pl': 'plk', + 'pt-br': 'ptb', + 'pt': 'ptg', + 'ru': 'rus', + 'sv-se': 'sve', + 'tr': 'trk' +}; + function sortLanguages(directoryNames) { return directoryNames.map(function (dirName) { var lower = dirName.toLowerCase(); @@ -288,3 +485,472 @@ function processNlsFiles(opts) { }); } exports.processNlsFiles = processNlsFiles; + +function prepareXlfFiles(projectName, extensionName) { + return event_stream_1.through(function (file) { + if (!file.isBuffer()) { + log('Error', "Failed to read component file: " + file.relative); + } + var extension = path.extname(file.path); + if (extension === '.json') { + var json = JSON.parse(file.contents.toString('utf8')); + if (BundledFormat.is(json)) { + importBundleJson(file, json, this); + } + else if (PackageJsonFormat.is(json) || ModuleJsonFormat.is(json)) { + importModuleOrPackageJson(file, json, projectName, this, extensionName); + } + else { + log('Error', 'JSON format cannot be deduced.'); + } + } + else if (extension === '.isl') { + importIsl(file, this); + } + }); +} +exports.prepareXlfFiles = prepareXlfFiles; +function getResource(sourceFile) { + var editorProject = 'vscode-editor', workbenchProject = 'vscode-workbench'; + var resource; + if (sourceFile.startsWith('vs/platform')) { + return { name: 'vs/platform', project: editorProject }; + } + else if (sourceFile.startsWith('vs/editor/contrib')) { + return { name: 'vs/editor/contrib', project: editorProject }; + } + else if (sourceFile.startsWith('vs/editor')) { + return { name: 'vs/editor', project: editorProject }; + } + else if (sourceFile.startsWith('vs/base')) { + return { name: 'vs/base', project: editorProject }; + } + else if (sourceFile.startsWith('vs/code')) { + return { name: 'vs/code', project: workbenchProject }; + } + else if (sourceFile.startsWith('vs/workbench/parts')) { + resource = sourceFile.split('/', 4).join('/'); + return { name: resource, project: workbenchProject }; + } + else if (sourceFile.startsWith('vs/workbench/services')) { + resource = sourceFile.split('/', 4).join('/'); + return { name: resource, project: workbenchProject }; + } + else if (sourceFile.startsWith('vs/workbench')) { + return { name: 'vs/workbench', project: workbenchProject }; + } + throw new Error("Could not identify the XLF bundle for " + sourceFile); +} +function importBundleJson(file, json, stream) { + var transifexEditorXlfs = Object.create(null); + for (var source in json.keys) { + var projectResource = getResource(source); + var resource = projectResource.name; + var project = projectResource.project; + var keys = json.keys[source]; + var messages = json.messages[source]; + if (keys.length !== messages.length) { + log('Error:', "There is a mismatch between keys and messages in " + file.relative); + } + var xlf = transifexEditorXlfs[resource] ? transifexEditorXlfs[resource] : transifexEditorXlfs[resource] = new XLF(project); + xlf.addFile(source, keys, messages); + } + for (var resource in transifexEditorXlfs) { + var newFilePath = transifexEditorXlfs[resource].project + "/" + resource.replace(/\//g, '_') + ".xlf"; + var xlfFile = new File({ path: newFilePath, contents: new Buffer(transifexEditorXlfs[resource].toString(), 'utf-8') }); + stream.emit('data', xlfFile); + } +} +// Keeps existing XLF instances and a state of how many files were already processed for faster file emission +var extensions = Object.create(null); +function importModuleOrPackageJson(file, json, projectName, stream, extensionName) { + if (ModuleJsonFormat.is(json) && json['keys'].length !== json['messages'].length) { + log('Error:', "There is a mismatch between keys and messages in " + file.relative); + } + // Prepare the source path for attribute in XLF & extract messages from JSON + var formattedSourcePath = file.relative.replace(/\\/g, '/'); + var messages = Object.keys(json).map(function (key) { return json[key].toString(); }); + // Stores the amount of localization files to be transformed to XLF before the emission + var localizationFilesCount, originalFilePath; + // If preparing XLF for external extension, then use different glob pattern and source path + if (extensionName) { + localizationFilesCount = glob.sync('**/*.nls.json').length; + originalFilePath = "" + formattedSourcePath.substr(0, formattedSourcePath.length - '.nls.json'.length); + } + else { + // Used for vscode/extensions folder + extensionName = formattedSourcePath.split('/')[0]; + localizationFilesCount = glob.sync("./extensions/" + extensionName + "/**/*.nls.json").length; + originalFilePath = "extensions/" + formattedSourcePath.substr(0, formattedSourcePath.length - '.nls.json'.length); + } + var extension = extensions[extensionName] ? + extensions[extensionName] : extensions[extensionName] = { xlf: new XLF(projectName), processed: 0 }; + if (ModuleJsonFormat.is(json)) { + extension.xlf.addFile(originalFilePath, json['keys'], json['messages']); + } + else { + extension.xlf.addFile(originalFilePath, Object.keys(json), messages); + } + // Check if XLF is populated with file nodes to emit it + if (++extensions[extensionName].processed === localizationFilesCount) { + var newFilePath = path.join(projectName, extensionName + '.xlf'); + var xlfFile = new File({ path: newFilePath, contents: new Buffer(extension.xlf.toString(), 'utf-8') }); + stream.emit('data', xlfFile); + } +} +var islXlf, islProcessed = 0; +function importIsl(file, stream) { + var islFiles = ['Default.isl', 'messages.en.isl']; + var projectName = 'vscode-workbench'; + var xlf = islXlf ? islXlf : islXlf = new XLF(projectName), keys = [], messages = []; + var model = new TextModel(file.contents.toString()); + var inMessageSection = false; + model.lines.forEach(function (line) { + if (line.length === 0) { + return; + } + var firstChar = line.charAt(0); + switch (firstChar) { + case ';': + // Comment line; + return; + case '[': + inMessageSection = '[Messages]' === line || '[CustomMessages]' === line; + return; + } + if (!inMessageSection) { + return; + } + var sections = line.split('='); + if (sections.length !== 2) { + log('Error:', "Badly formatted message found: " + line); + } + else { + var key = sections[0]; + var value = sections[1]; + if (key.length > 0 && value.length > 0) { + keys.push(key); + messages.push(value); + } + } + }); + var originalPath = file.path.substring(file.cwd.length + 1, file.path.split('.')[0].length).replace(/\\/g, '/'); + xlf.addFile(originalPath, keys, messages); + // Emit only upon all ISL files combined into single XLF instance + if (++islProcessed === islFiles.length) { + var newFilePath = path.join(projectName, 'setup.xlf'); + var xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8') }); + stream.emit('data', xlfFile); + } +} +function pushXlfFiles(apiUrl, username, password) { + return event_stream_1.through(function (file) { + var project = path.dirname(file.relative); + var fileName = path.basename(file.path); + var slug = fileName.substr(0, fileName.length - '.xlf'.length); + var credentials = { + 'user': username, + 'password': password + }; + // Check if resource already exists, if not, then create it. + tryGetResource(project, slug, apiUrl, credentials).then(function (exists) { + if (exists) { + updateResource(project, slug, file, apiUrl, credentials); + } + else { + createResource(project, slug, file, apiUrl, credentials); + } + }); + }); +} +exports.pushXlfFiles = pushXlfFiles; +function tryGetResource(project, slug, apiUrl, credentials) { + return new Promise(function (resolve, reject) { + var url = apiUrl + "/project/" + project + "/resource/" + slug + "/?details"; + request.get(url, { 'auth': credentials }).on('response', function (response) { + if (response.statusCode === 404) { + resolve(false); + } + else if (response.statusCode === 200) { + resolve(true); + } + else { + reject("Failed to query resource " + slug + ". Response: " + response.statusCode + " " + response.statusMessage); + } + }); + }); +} +function createResource(project, slug, xlfFile, apiUrl, credentials) { + var url = apiUrl + "/project/" + project + "/resources"; + var options = { + 'body': { + 'content': xlfFile.contents.toString(), + 'name': slug, + 'slug': slug, + 'i18n_type': 'XLIFF' + }, + 'json': true, + 'auth': credentials + }; + request.post(url, options, function (err, res) { + if (err) { + log('Error:', "Failed to create Transifex " + project + "/" + slug + ": " + err); + } + if (res.statusCode === 201) { + log("Resource " + project + "/" + slug + " successfully created on Transifex."); + } + else { + log('Error:', "Something went wrong creating " + slug + " in " + project + ". " + res.statusCode); + } + }); +} +/** + * The following link provides information about how Transifex handles updates of a resource file: + * https://dev.befoolish.co/tx-docs/public/projects/updating-content#what-happens-when-you-update-files + */ +function updateResource(project, slug, xlfFile, apiUrl, credentials) { + var url = apiUrl + "/project/" + project + "/resource/" + slug + "/content"; + var options = { + 'body': { 'content': xlfFile.contents.toString() }, + 'json': true, + 'auth': credentials + }; + request.put(url, options, function (err, res, body) { + if (err) { + log('Error:', "Failed to update Transifex " + project + "/" + slug + ": " + err); + } + if (res.statusCode === 200) { + log("Resource " + project + "/" + slug + " successfully updated on Transifex. Strings added: " + body['strings_added'] + ", updated: " + body['strings_updated'] + ", deleted: " + body['strings_delete']); + } + else { + log('Error:', "Something went wrong updating " + slug + " in " + project + ". " + res.statusCode); + } + }); +} +function getMetadataResources(pathToMetadata) { + var metadata = fs.readFileSync(pathToMetadata).toString('utf8'); + var json = JSON.parse(metadata); + var slugs = []; + var _loop_1 = function (source) { + var projectResource = getResource(source); + if (!slugs.find(function (slug) { return slug.name === projectResource.name && slug.project === projectResource.project; })) { + slugs.push(projectResource); + } + }; + for (var source in json['keys']) { + _loop_1(source); + } + return slugs; +} +function obtainProjectResources(projectName) { + var resources; + if (projectName === 'vscode-editor-workbench') { + resources = getMetadataResources('./out-vscode/nls.metadata.json'); + resources.push({ name: 'setup', project: 'vscode-workbench' }); + } + else if (projectName === 'vscode-extensions') { + var extensionsToLocalize = glob.sync('./extensions/**/*.nls.json').map(function (extension) { return extension.split('/')[2]; }); + var resourcesToPull_1 = []; + resources = []; + extensionsToLocalize.forEach(function (extension) { + if (resourcesToPull_1.indexOf(extension) === -1) { + resourcesToPull_1.push(extension); + resources.push({ name: extension, project: projectName }); + } + }); + } + return resources; +} +function pullXlfFiles(projectName, apiUrl, username, password, resources) { + if (!resources) { + resources = obtainProjectResources(projectName); + } + if (!resources) { + throw new Error('Transifex projects and resources must be defined to be able to pull translations from Transifex.'); + } + var credentials = { + 'auth': { + 'user': username, + 'password': password + } + }; + var expectedTranslationsCount = vscodeLanguages.length * resources.length; + var translationsRetrieved = 0, called = false; + return es.readable(function (count, callback) { + // Mark end of stream when all resources were retrieved + if (translationsRetrieved === expectedTranslationsCount) { + return this.emit('end'); + } + if (!called) { + called = true; + var stream_1 = this; + vscodeLanguages.map(function (language) { + resources.map(function (resource) { + var slug = resource.name.replace(/\//g, '_'); + var project = resource.project; + var iso639 = iso639_3_to_2[language]; + var url = apiUrl + "/project/" + project + "/resource/" + slug + "/translation/" + iso639 + "?file&mode=onlyreviewed"; + var xlfBuffer = '', responseCode; + request.get(url, credentials) + .on('response', function (response) { + responseCode = response.statusCode; + }) + .on('data', function (data) { return xlfBuffer += data; }) + .on('end', function () { + if (responseCode === 200) { + stream_1.emit('data', new File({ contents: new Buffer(xlfBuffer) })); + } + else { + log('Error:', slug + " in " + project + " returned no data. Response code: " + responseCode + "."); + } + translationsRetrieved++; + }) + .on('error', function (error) { + log('Error:', "Failed to query resource " + slug + " with the following error: " + error); + }); + }); + }); + } + callback(); + }); +} +exports.pullXlfFiles = pullXlfFiles; +function prepareJsonFiles() { + return event_stream_1.through(function (xlf) { + var stream = this; + XLF.parse(xlf.contents.toString()).then(function (resolvedFiles) { + resolvedFiles.forEach(function (file) { + var messages = file.messages, translatedFile; + // ISL file path always starts with 'build/' + if (file.originalFilePath.startsWith('build/')) { + var defaultLanguages = { 'zh-cn': true, 'zh-tw': true, 'ko': true }; + if (path.basename(file.originalFilePath) === 'Default' && !defaultLanguages[file.language]) { + return; + } + translatedFile = createIslFile('..', file.originalFilePath, messages, iso639_2_to_3[file.language]); + } + else { + translatedFile = createI18nFile(iso639_2_to_3[file.language], file.originalFilePath, messages); + } + stream.emit('data', translatedFile); + }); + }, function (rejectReason) { + log('Error:', rejectReason); + }); + }); +} +exports.prepareJsonFiles = prepareJsonFiles; +function createI18nFile(base, originalFilePath, messages) { + var content = [ + '/*---------------------------------------------------------------------------------------------', + ' * Copyright (c) Microsoft Corporation. All rights reserved.', + ' * Licensed under the MIT License. See License.txt in the project root for license information.', + ' *--------------------------------------------------------------------------------------------*/', + '// Do not edit this file. It is machine generated.' + ].join('\n') + '\n' + JSON.stringify(messages, null, '\t').replace(/\r\n/g, '\n'); + return new File({ + path: path.join(base, originalFilePath + '.i18n.json'), + contents: new Buffer(content, 'utf8') + }); +} +exports.createI18nFile = createI18nFile; +var languageNames = { + 'chs': 'Simplified Chinese', + 'cht': 'Traditional Chinese', + 'kor': 'Korean' +}; +var languageIds = { + 'chs': '$0804', + 'cht': '$0404', + 'kor': '$0412' +}; +var encodings = { + 'chs': 'CP936', + 'cht': 'CP950', + 'jpn': 'CP932', + 'kor': 'CP949', + 'deu': 'CP1252', + 'fra': 'CP1252', + 'esn': 'CP1252', + 'rus': 'CP1251', + 'ita': 'CP1252' +}; +function createIslFile(base, originalFilePath, messages, language) { + var content = []; + var originalContent; + if (path.basename(originalFilePath) === 'Default') { + originalContent = new TextModel(fs.readFileSync(originalFilePath + '.isl', 'utf8')); + } + else { + originalContent = new TextModel(fs.readFileSync(originalFilePath + '.en.isl', 'utf8')); + } + originalContent.lines.forEach(function (line) { + if (line.length > 0) { + var firstChar = line.charAt(0); + if (firstChar === '[' || firstChar === ';') { + if (line === '; *** Inno Setup version 5.5.3+ English messages ***') { + content.push("; *** Inno Setup version 5.5.3+ " + languageNames[language] + " messages ***"); + } + else { + content.push(line); + } + } + else { + var sections = line.split('='); + var key = sections[0]; + var translated = line; + if (key) { + if (key === 'LanguageName') { + translated = key + "=" + languageNames[language]; + } + else if (key === 'LanguageID') { + translated = key + "=" + languageIds[language]; + } + else if (key === 'LanguageCodePage') { + translated = key + "=" + encodings[language].substr(2); + } + else { + var translatedMessage = messages[key]; + if (translatedMessage) { + translated = key + "=" + translatedMessage; + } + } + } + content.push(translated); + } + } + }); + var tag = iso639_3_to_2[language]; + var basename = path.basename(originalFilePath); + var filePath = path.join(base, path.dirname(originalFilePath), basename) + "." + tag + ".isl"; + return new File({ + path: filePath, + contents: iconv.encode(new Buffer(content.join('\r\n'), 'utf8'), encodings[language]) + }); +} +exports.createIslFile = createIslFile; +function encodeEntities(value) { + var result = []; + for (var i = 0; i < value.length; i++) { + var ch = value[i]; + switch (ch) { + case '<': + result.push('<'); + break; + case '>': + result.push('>'); + break; + case '&': + result.push('&'); + break; + default: + result.push(ch); + } + } + return result.join(''); +} +function decodeEntities(value) { + return value.replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&'); +} +exports.decodeEntities = decodeEntities; +//# sourceMappingURL=i18n.js.map diff --git a/build/lib/i18n.ts b/build/lib/i18n.ts index 046cd8c8d8b..146664b6a34 100644 --- a/build/lib/i18n.ts +++ b/build/lib/i18n.ts @@ -10,8 +10,13 @@ import { through } from 'event-stream'; import { ThroughStream } from 'through'; import File = require('vinyl'); import * as Is from 'is'; +import xml2js = require('xml2js'); +import * as glob from 'glob'; var util = require('gulp-util'); +const request = require('request'); +const es = require('event-stream'); +var iconv = require('iconv-lite'); function log(message: any, ...rest: any[]): void { util.log(util.colors.green('[i18n]'), message, ...rest); @@ -21,6 +26,17 @@ interface Map { [key: string]: V; } +interface Item { + id: string; + message: string; + comment: string; +} + +interface Resource { + name: string; + project: string; +} + interface LocalizeInfo { key: string; comment: string[]; @@ -52,6 +68,205 @@ module BundledFormat { } } +interface ValueFormat { + message: string; + comment: string[]; +} + +interface PackageJsonFormat { + [key: string]: string | ValueFormat; +} + +module PackageJsonFormat { + export function is(value: any): value is PackageJsonFormat { + if (Is.undef(value) || !Is.object(value)) { + return false; + } + return Object.keys(value).every(key => { + let element = value[key]; + return Is.string(element) || (Is.object(element) && Is.defined(element.message) && Is.defined(element.comment)); + }); + } +} + +interface ModuleJsonFormat { + messages: string[]; + keys: (string | LocalizeInfo)[]; +} + +module ModuleJsonFormat { + export function is(value: any): value is ModuleJsonFormat { + let candidate = value as ModuleJsonFormat; + return Is.defined(candidate) + && Is.array(candidate.messages) && candidate.messages.every(message => Is.string(message)) + && Is.array(candidate.keys) && candidate.keys.every(key => Is.string(key) || LocalizeInfo.is(key)); + } +} + +export class Line { + private buffer: string[] = []; + + constructor(private indent: number = 0) { + if (indent > 0) { + this.buffer.push(new Array(indent + 1).join(' ')); + } + } + + public append(value: string): Line { + this.buffer.push(value); + return this; + } + + public toString(): string { + return this.buffer.join(''); + } +} + +class TextModel { + private _lines: string[]; + + constructor(contents: string) { + this._lines = contents.split(/\r\n|\r|\n/); + } + + public get lines(): string[] { + return this._lines; + } +} + +export class XLF { + private buffer: string[]; + private files: Map; + + constructor(public project: string) { + this.buffer = []; + this.files = Object.create(null); + } + + public toString(): string { + this.appendHeader(); + + for (let file in this.files) { + this.appendNewLine(``, 2); + for (let item of this.files[file]) { + this.addStringItem(item); + } + this.appendNewLine('', 2); + } + + this.appendFooter(); + return this.buffer.join('\r\n'); + } + + public addFile(original: string, keys: any[], messages: string[]) { + this.files[original] = []; + let existingKeys = []; + + for (let key of keys) { + // Ignore duplicate keys because Transifex does not populate those with translated values. + if (existingKeys.indexOf(key) !== -1) { + continue; + } + existingKeys.push(key); + + let message: string = encodeEntities(messages[keys.indexOf(key)]); + let comment: string = undefined; + + // Check if the message contains description (if so, it becomes an object type in JSON) + if (Is.string(key)) { + this.files[original].push({ id: key, message: message, comment: comment }); + } else { + if (key['comment'] && key['comment'].length > 0) { + comment = key['comment'].map(comment => encodeEntities(comment)).join('\r\n'); + } + + this.files[original].push({ id: key['key'], message: message, comment: comment }); + } + } + } + + private addStringItem(item: Item): void { + if (!item.id || !item.message) { + //throw new Error('No item ID or value specified.'); + } + + this.appendNewLine(``, 4); + this.appendNewLine(`${encodeEntities(item.message)}`, 6); + + if (item.comment) { + this.appendNewLine(`${item.comment}`, 6); + } + + this.appendNewLine('', 4); + } + + private appendHeader(): void { + this.appendNewLine('', 0); + this.appendNewLine('', 0); + } + + private appendFooter(): void { + this.appendNewLine('', 0); + } + + private appendNewLine(content: string, indent?: number): void { + let line = new Line(indent); + line.append(content); + this.buffer.push(line.toString()); + } + + static parse = function(xlfString: string) : Promise<{ messages: Map, originalFilePath: string, language: string }[]> { + return new Promise((resolve, reject) => { + let parser = new xml2js.Parser(); + + let files: { messages: Map, originalFilePath: string, language: string }[] = []; + + parser.parseString(xlfString, function(err, result) { + if (err) { + reject(`Failed to parse XLIFF string. ${err}`); + } + + const fileNodes: any[] = result['xliff']['file']; + if (!fileNodes) { + reject('XLIFF file does not contain "xliff" or "file" node(s) required for parsing.'); + } + + fileNodes.forEach((file) => { + const originalFilePath = file.$.original; + if (!originalFilePath) { + reject('XLIFF file node does not contain original attribute to determine the original location of the resource file.'); + } + const language = file.$['target-language'].toLowerCase(); + if (!language) { + reject('XLIFF file node does not contain target-language attribute to determine translated language.'); + } + + let messages: Map = {}; + const transUnits = file.body[0]['trans-unit']; + + transUnits.forEach(unit => { + const key = unit.$.id; + if (!unit.target) { + return; // No translation available + } + + const val = unit.target.toString(); + if (key && val) { + messages[key] = decodeEntities(val); + } else { + reject('XLIFF file does not contain full localization data. ID or target translation for one of the trans-unit nodes is not present.'); + } + }); + + files.push({ messages: messages, originalFilePath: originalFilePath, language: language }); + }); + + resolve(files); + }); + }); + }; +} + const vscodeLanguages: string[] = [ 'chs', 'cht', @@ -85,6 +300,26 @@ const iso639_3_to_2: Map = { 'trk': 'tr' }; +const iso639_2_to_3: Map = { + 'zh-cn': 'chs', + 'zh-tw': 'cht', + 'cs-cz': 'csy', + 'de': 'deu', + 'en': 'enu', + 'es': 'esn', + 'fr': 'fra', + 'hu': 'hun', + 'it': 'ita', + 'ja': 'jpn', + 'ko': 'kor', + 'nl': 'nld', + 'pl': 'plk', + 'pt-br': 'ptb', + 'pt': 'ptg', + 'ru': 'rus', + 'sv-se': 'sve', + 'tr': 'trk' +}; interface IDirectoryInfo { name: string; iso639_2: string; @@ -138,7 +373,7 @@ function stripComments(content: string): string { } }); return result; -}; +} function escapeCharacters(value:string):string { var result:string[] = []; @@ -308,4 +543,509 @@ export function processNlsFiles(opts:{fileHeader:string;}): ThroughStream { } this.emit('data', file); }); +} + +export function prepareXlfFiles(projectName?: string, extensionName?: string): ThroughStream { + return through( + function (file: File) { + if (!file.isBuffer()) { + log('Error', `Failed to read component file: ${file.relative}`); + } + + const extension = path.extname(file.path); + if (extension === '.json') { + const json = JSON.parse((file.contents).toString('utf8')); + + if (BundledFormat.is(json)) { + importBundleJson(file, json, this); + } else if (PackageJsonFormat.is(json) || ModuleJsonFormat.is(json)) { + importModuleOrPackageJson(file, json, projectName, this, extensionName); + } else { + log('Error', 'JSON format cannot be deduced.'); + } + } else if (extension === '.isl') { + importIsl(file, this); + } + } + ); +} + +function getResource(sourceFile: string): Resource { + const editorProject: string = 'vscode-editor', + workbenchProject: string = 'vscode-workbench'; + let resource: string; + + if (sourceFile.startsWith('vs/platform')) { + return { name: 'vs/platform', project: editorProject }; + } else if (sourceFile.startsWith('vs/editor/contrib')) { + return { name: 'vs/editor/contrib', project: editorProject }; + } else if (sourceFile.startsWith('vs/editor')) { + return { name: 'vs/editor', project: editorProject }; + } else if (sourceFile.startsWith('vs/base')) { + return { name: 'vs/base', project: editorProject }; + } else if (sourceFile.startsWith('vs/code')) { + return { name: 'vs/code', project: workbenchProject }; + } else if (sourceFile.startsWith('vs/workbench/parts')) { + resource = sourceFile.split('/', 4).join('/'); + return { name: resource, project: workbenchProject }; + } else if (sourceFile.startsWith('vs/workbench/services')) { + resource = sourceFile.split('/', 4).join('/'); + return { name: resource, project: workbenchProject }; + } else if (sourceFile.startsWith('vs/workbench')) { + return { name: 'vs/workbench', project: workbenchProject }; + } + + throw new Error (`Could not identify the XLF bundle for ${sourceFile}`); +} + + +function importBundleJson(file: File, json: BundledFormat, stream: ThroughStream): void { + let transifexEditorXlfs: Map = Object.create(null); + + for (let source in json.keys) { + const projectResource = getResource(source); + const resource = projectResource.name; + const project = projectResource.project; + + const keys = json.keys[source]; + const messages = json.messages[source]; + if (keys.length !== messages.length) { + log('Error:', `There is a mismatch between keys and messages in ${file.relative}`); + } + + let xlf = transifexEditorXlfs[resource] ? transifexEditorXlfs[resource] : transifexEditorXlfs[resource] = new XLF(project); + xlf.addFile(source, keys, messages); + } + + for (let resource in transifexEditorXlfs) { + const newFilePath = `${transifexEditorXlfs[resource].project}/${resource.replace(/\//g, '_')}.xlf`; + const xlfFile = new File({ path: newFilePath, contents: new Buffer(transifexEditorXlfs[resource].toString(), 'utf-8')}); + stream.emit('data', xlfFile); + } +} + +// Keeps existing XLF instances and a state of how many files were already processed for faster file emission +var extensions: Map<{ xlf: XLF, processed: number }> = Object.create(null); +function importModuleOrPackageJson(file: File, json: ModuleJsonFormat | PackageJsonFormat, projectName: string, stream: ThroughStream, extensionName?: string): void { + if (ModuleJsonFormat.is(json) && json['keys'].length !== json['messages'].length) { + log('Error:', `There is a mismatch between keys and messages in ${file.relative}`); + } + + // Prepare the source path for attribute in XLF & extract messages from JSON + const formattedSourcePath = file.relative.replace(/\\/g, '/'); + const messages = Object.keys(json).map((key) => json[key].toString()); + + // Stores the amount of localization files to be transformed to XLF before the emission + let localizationFilesCount, + originalFilePath; + // If preparing XLF for external extension, then use different glob pattern and source path + if (extensionName) { + localizationFilesCount = glob.sync('**/*.nls.json').length; + originalFilePath = `${formattedSourcePath.substr(0, formattedSourcePath.length - '.nls.json'.length)}`; + } else { + // Used for vscode/extensions folder + extensionName = formattedSourcePath.split('/')[0]; + localizationFilesCount = glob.sync(`./extensions/${extensionName}/**/*.nls.json`).length; + originalFilePath = `extensions/${formattedSourcePath.substr(0, formattedSourcePath.length - '.nls.json'.length)}`; + } + + let extension = extensions[extensionName] ? + extensions[extensionName] : extensions[extensionName] = { xlf: new XLF(projectName), processed: 0 }; + + if (ModuleJsonFormat.is(json)) { + extension.xlf.addFile(originalFilePath, json['keys'], json['messages']); + } else { + extension.xlf.addFile(originalFilePath, Object.keys(json), messages); + } + + // Check if XLF is populated with file nodes to emit it + if (++extensions[extensionName].processed === localizationFilesCount) { + const newFilePath = path.join(projectName, extensionName + '.xlf'); + const xlfFile = new File({ path: newFilePath, contents: new Buffer(extension.xlf.toString(), 'utf-8')}); + stream.emit('data', xlfFile); + } +} + +var islXlf: XLF, + islProcessed: number = 0; + +function importIsl(file: File, stream: ThroughStream) { + const islFiles = ['Default.isl', 'messages.en.isl']; + const projectName = 'vscode-workbench'; + + let xlf = islXlf ? islXlf : islXlf = new XLF(projectName), + keys: string[] = [], + messages: string[] = []; + + let model = new TextModel(file.contents.toString()); + let inMessageSection = false; + model.lines.forEach(line => { + if (line.length === 0) { + return; + } + let firstChar = line.charAt(0); + switch (firstChar) { + case ';': + // Comment line; + return; + case '[': + inMessageSection = '[Messages]' === line || '[CustomMessages]' === line; + return; + } + if (!inMessageSection) { + return; + } + let sections: string[] = line.split('='); + if (sections.length !== 2) { + log('Error:', `Badly formatted message found: ${line}`); + } else { + let key = sections[0]; + let value = sections[1]; + if (key.length > 0 && value.length > 0) { + keys.push(key); + messages.push(value); + } + } + }); + + const originalPath = file.path.substring(file.cwd.length+1, file.path.split('.')[0].length).replace(/\\/g, '/'); + xlf.addFile(originalPath, keys, messages); + + // Emit only upon all ISL files combined into single XLF instance + if (++islProcessed === islFiles.length) { + const newFilePath = path.join(projectName, 'setup.xlf'); + const xlfFile = new File({ path: newFilePath, contents: new Buffer(xlf.toString(), 'utf-8')}); + stream.emit('data', xlfFile); + } +} + +export function pushXlfFiles(apiUrl: string, username: string, password: string): ThroughStream { + return through(function(file: File) { + const project = path.dirname(file.relative); + const fileName = path.basename(file.path); + const slug = fileName.substr(0, fileName.length - '.xlf'.length); + const credentials = { + 'user': username, + 'password': password + }; + + // Check if resource already exists, if not, then create it. + tryGetResource(project, slug, apiUrl, credentials).then(exists => { + if (exists) { + updateResource(project, slug, file, apiUrl, credentials); + } else { + createResource(project, slug, file, apiUrl, credentials); + } + }); + }); +} + +function tryGetResource(project: string, slug: string, apiUrl: string, credentials: any): Promise { + return new Promise((resolve, reject) => { + const url = `${apiUrl}/project/${project}/resource/${slug}/?details`; + request.get(url, { 'auth': credentials }).on('response', function (response) { + if (response.statusCode === 404) { + resolve(false); + } else if (response.statusCode === 200) { + resolve(true); + } else { + reject(`Failed to query resource ${slug}. Response: ${response.statusCode} ${response.statusMessage}`); + } + }); + }); +} + +function createResource(project: string, slug: string, xlfFile: File, apiUrl:string, credentials: any): void { + const url = `${apiUrl}/project/${project}/resources`; + const options = { + 'body': { + 'content': xlfFile.contents.toString(), + 'name': slug, + 'slug': slug, + 'i18n_type': 'XLIFF' + }, + 'json': true, + 'auth': credentials + }; + + request.post(url, options, function(err, res) { + if (err) { + log('Error:', `Failed to create Transifex ${project}/${slug}: ${err}`); + } + + if (res.statusCode === 201) { + log(`Resource ${project}/${slug} successfully created on Transifex.`); + } else { + log('Error:', `Something went wrong creating ${slug} in ${project}. ${res.statusCode}`); + } + }); +} + +/** + * The following link provides information about how Transifex handles updates of a resource file: + * https://dev.befoolish.co/tx-docs/public/projects/updating-content#what-happens-when-you-update-files + */ +function updateResource(project: string, slug: string, xlfFile: File, apiUrl: string, credentials: any) : void { + const url = `${apiUrl}/project/${project}/resource/${slug}/content`; + const options = { + 'body': { 'content': xlfFile.contents.toString() }, + 'json': true, + 'auth': credentials + }; + + request.put(url, options, function(err, res, body) { + if (err) { + log('Error:', `Failed to update Transifex ${project}/${slug}: ${err}`); + } + + if (res.statusCode === 200) { + log(`Resource ${project}/${slug} successfully updated on Transifex. Strings added: ${body['strings_added']}, updated: ${body['strings_updated']}, deleted: ${body['strings_delete']}`); + } else { + log('Error:', `Something went wrong updating ${slug} in ${project}. ${res.statusCode}`); + } + }); +} + +function getMetadataResources(pathToMetadata: string) : Resource[] { + const metadata = fs.readFileSync(pathToMetadata).toString('utf8'); + const json = JSON.parse(metadata); + let slugs = []; + + for (let source in json['keys']) { + let projectResource = getResource(source); + if (!slugs.find(slug => slug.name === projectResource.name && slug.project === projectResource.project)) { + slugs.push(projectResource); + } + } + + return slugs; +} + +function obtainProjectResources(projectName: string): Resource[] { + let resources: Resource[]; + + if (projectName === 'vscode-editor-workbench') { + resources = getMetadataResources('./out-vscode/nls.metadata.json'); + resources.push({ name: 'setup', project: 'vscode-workbench' }); + } else if (projectName === 'vscode-extensions') { + let extensionsToLocalize: string[] = glob.sync('./extensions/**/*.nls.json').map(extension => extension.split('/')[2]); + let resourcesToPull: string[] = []; + resources = []; + + extensionsToLocalize.forEach(extension => { + if (resourcesToPull.indexOf(extension) === -1) { // remove duplicate elements returned by glob + resourcesToPull.push(extension); + resources.push({ name: extension, project: projectName }); + } + }); + } + + return resources; +} + +export function pullXlfFiles(projectName: string, apiUrl: string, username: string, password: string, resources?: Resource[]): NodeJS.ReadableStream { + if (!resources) { + resources = obtainProjectResources(projectName); + } + if (!resources) { + throw new Error('Transifex projects and resources must be defined to be able to pull translations from Transifex.'); + } + + const credentials = { + 'auth': { + 'user': username, + 'password': password + } + }; + let expectedTranslationsCount = vscodeLanguages.length * resources.length; + let translationsRetrieved = 0, called = false; + + return es.readable(function(count, callback) { + // Mark end of stream when all resources were retrieved + if (translationsRetrieved === expectedTranslationsCount) { + return this.emit('end'); + } + + if (!called) { + called = true; + const stream = this; + + vscodeLanguages.map(function(language) { + resources.map(function (resource) { + const slug = resource.name.replace(/\//g, '_'); + const project = resource.project; + const iso639 = iso639_3_to_2[language]; + const url = `${apiUrl}/project/${project}/resource/${slug}/translation/${iso639}?file&mode=onlyreviewed`; + + let xlfBuffer: string = '', responseCode: number; + request.get(url, credentials) + .on('response', (response) => { + responseCode = response.statusCode; + }) + .on('data', (data) => xlfBuffer += data) + .on('end', () => { + if (responseCode === 200) { + stream.emit('data', new File({ contents: new Buffer(xlfBuffer) })); + } else { + log('Error:', `${slug} in ${project} returned no data. Response code: ${responseCode}.`); + } + translationsRetrieved++; + }) + .on('error', (error) => { + log('Error:', `Failed to query resource ${slug} with the following error: ${error}`); + }); + }); + }); + } + + callback(); + }); +} + +export function prepareJsonFiles(): ThroughStream { + return through(function(xlf: File) { + let stream = this; + + XLF.parse(xlf.contents.toString()).then( + function(resolvedFiles) { + resolvedFiles.forEach(file => { + let messages = file.messages, translatedFile; + + // ISL file path always starts with 'build/' + if (file.originalFilePath.startsWith('build/')) { + const defaultLanguages = { 'zh-cn': true, 'zh-tw': true, 'ko': true }; + if (path.basename(file.originalFilePath) === 'Default' && !defaultLanguages[file.language]) { + return; + } + + translatedFile = createIslFile('..', file.originalFilePath, messages, iso639_2_to_3[file.language]); + } else { + translatedFile = createI18nFile(iso639_2_to_3[file.language], file.originalFilePath, messages); + } + + stream.emit('data', translatedFile); + }); + }, + function(rejectReason) { + log('Error:', rejectReason); + } + ); + }); +} + +export function createI18nFile(base: string, originalFilePath: string, messages: Map): File { + let content = [ + '/*---------------------------------------------------------------------------------------------', + ' * Copyright (c) Microsoft Corporation. All rights reserved.', + ' * Licensed under the MIT License. See License.txt in the project root for license information.', + ' *--------------------------------------------------------------------------------------------*/', + '// Do not edit this file. It is machine generated.' + ].join('\n') + '\n' + JSON.stringify(messages, null, '\t').replace(/\r\n/g, '\n'); + + return new File({ + path: path.join(base, originalFilePath + '.i18n.json'), + contents: new Buffer(content, 'utf8') + }); +} + + +const languageNames: Map = { + 'chs': 'Simplified Chinese', + 'cht': 'Traditional Chinese', + 'kor': 'Korean' +}; + +const languageIds: Map = { + 'chs': '$0804', + 'cht': '$0404', + 'kor': '$0412' +}; + +const encodings: Map = { + 'chs': 'CP936', + 'cht': 'CP950', + 'jpn': 'CP932', + 'kor': 'CP949', + 'deu': 'CP1252', + 'fra': 'CP1252', + 'esn': 'CP1252', + 'rus': 'CP1251', + 'ita': 'CP1252' +}; + +export function createIslFile(base: string, originalFilePath: string, messages: Map, language: string): File { + let content: string[] = []; + let originalContent: TextModel; + if (path.basename(originalFilePath) === 'Default') { + originalContent = new TextModel(fs.readFileSync(originalFilePath + '.isl', 'utf8')); + } else { + originalContent = new TextModel(fs.readFileSync(originalFilePath + '.en.isl', 'utf8')); + } + + originalContent.lines.forEach(line => { + if (line.length > 0) { + let firstChar = line.charAt(0); + if (firstChar === '[' || firstChar === ';') { + if (line === '; *** Inno Setup version 5.5.3+ English messages ***') { + content.push(`; *** Inno Setup version 5.5.3+ ${languageNames[language]} messages ***`); + } else { + content.push(line); + } + } else { + let sections: string[] = line.split('='); + let key = sections[0]; + let translated = line; + if (key) { + if (key === 'LanguageName') { + translated = `${key}=${languageNames[language]}`; + } else if (key === 'LanguageID') { + translated = `${key}=${languageIds[language]}`; + } else if (key === 'LanguageCodePage') { + translated = `${key}=${encodings[language].substr(2)}`; + } else { + let translatedMessage = messages[key]; + if (translatedMessage) { + translated = `${key}=${translatedMessage}`; + } + } + } + + content.push(translated); + } + } + }); + + let tag = iso639_3_to_2[language]; + let basename = path.basename(originalFilePath); + let filePath = `${path.join(base, path.dirname(originalFilePath), basename)}.${tag}.isl`; + + return new File({ + path: filePath, + contents: iconv.encode(new Buffer(content.join('\r\n'), 'utf8'), encodings[language]) + }); +} + +function encodeEntities(value: string): string { + var result: string[] = []; + for (var i = 0; i < value.length; i++) { + var ch = value[i]; + switch (ch) { + case '<': + result.push('<'); + break; + case '>': + result.push('>'); + break; + case '&': + result.push('&'); + break; + default: + result.push(ch); + } + } + return result.join(''); +} + +export function decodeEntities(value:string): string { + return value.replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&'); } \ No newline at end of file diff --git a/extensions/typescript/package.json b/extensions/typescript/package.json index c5721ef1c0c..d93d295c725 100644 --- a/extensions/typescript/package.json +++ b/extensions/typescript/package.json @@ -403,4 +403,4 @@ } ] } -} \ No newline at end of file +} From c47a8c36356545b90318b51c6d04b2cf94ff614c Mon Sep 17 00:00:00 2001 From: Michel Kaporin Date: Fri, 17 Mar 2017 15:27:05 +0100 Subject: [PATCH 3/5] Added i18n tests. --- build/lib/i18n.js | 13 +++------ build/lib/i18n.ts | 6 ++--- build/lib/test/i18n.test.js | 41 ++++++++++++++++++++++++++++ build/lib/test/i18n.test.ts | 54 +++++++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 13 deletions(-) create mode 100644 build/lib/test/i18n.test.js create mode 100644 build/lib/test/i18n.test.ts diff --git a/build/lib/i18n.js b/build/lib/i18n.js index a8256d1ad37..7f602efdd3b 100644 --- a/build/lib/i18n.js +++ b/build/lib/i18n.js @@ -9,14 +9,12 @@ var fs = require("fs"); var event_stream_1 = require("event-stream"); var File = require("vinyl"); var Is = require("is"); -var util = require('gulp-util'); var xml2js = require("xml2js"); var glob = require("glob"); var util = require('gulp-util'); var request = require('request'); var es = require('event-stream'); var iconv = require('iconv-lite'); - function log(message) { var rest = []; for (var _i = 1; _i < arguments.length; _i++) { @@ -44,7 +42,6 @@ var BundledFormat; } BundledFormat.is = is; })(BundledFormat || (BundledFormat = {})); - var PackageJsonFormat; (function (PackageJsonFormat) { function is(value) { @@ -148,7 +145,7 @@ var XLF = (function () { //throw new Error('No item ID or value specified.'); } this.appendNewLine("", 4); - this.appendNewLine("" + encodeEntities(item.message) + "", 6); + this.appendNewLine("" + item.message + "", 6); if (item.comment) { this.appendNewLine("" + item.comment + "", 6); } @@ -211,7 +208,6 @@ XLF.parse = function (xlfString) { }); }; exports.XLF = XLF; - var vscodeLanguages = [ 'chs', 'cht', @@ -243,7 +239,6 @@ var iso639_3_to_2 = { 'sve': 'sv-se', 'trk': 'tr' }; - var iso639_2_to_3 = { 'zh-cn': 'chs', 'zh-tw': 'cht', @@ -264,7 +259,6 @@ var iso639_2_to_3 = { 'sv-se': 'sve', 'tr': 'trk' }; - function sortLanguages(directoryNames) { return directoryNames.map(function (dirName) { var lower = dirName.toLowerCase(); @@ -316,7 +310,6 @@ function stripComments(content) { }); return result; } -; function escapeCharacters(value) { var result = []; for (var i = 0; i < value.length; i++) { @@ -485,7 +478,6 @@ function processNlsFiles(opts) { }); } exports.processNlsFiles = processNlsFiles; - function prepareXlfFiles(projectName, extensionName) { return event_stream_1.through(function (file) { if (!file.isBuffer()) { @@ -541,6 +533,7 @@ function getResource(sourceFile) { } throw new Error("Could not identify the XLF bundle for " + sourceFile); } +exports.getResource = getResource; function importBundleJson(file, json, stream) { var transifexEditorXlfs = Object.create(null); for (var source in json.keys) { @@ -953,4 +946,4 @@ function decodeEntities(value) { return value.replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&'); } exports.decodeEntities = decodeEntities; -//# sourceMappingURL=i18n.js.map +//# sourceMappingURL=i18n.js.map \ No newline at end of file diff --git a/build/lib/i18n.ts b/build/lib/i18n.ts index 146664b6a34..9c5f1b89698 100644 --- a/build/lib/i18n.ts +++ b/build/lib/i18n.ts @@ -32,7 +32,7 @@ interface Item { comment: string; } -interface Resource { +export interface Resource { name: string; project: string; } @@ -191,7 +191,7 @@ export class XLF { } this.appendNewLine(``, 4); - this.appendNewLine(`${encodeEntities(item.message)}`, 6); + this.appendNewLine(`${item.message}`, 6); if (item.comment) { this.appendNewLine(`${item.comment}`, 6); @@ -570,7 +570,7 @@ export function prepareXlfFiles(projectName?: string, extensionName?: string): T ); } -function getResource(sourceFile: string): Resource { +export function getResource(sourceFile: string): Resource { const editorProject: string = 'vscode-editor', workbenchProject: string = 'vscode-workbench'; let resource: string; diff --git a/build/lib/test/i18n.test.js b/build/lib/test/i18n.test.js new file mode 100644 index 00000000000..4e7b60b459e --- /dev/null +++ b/build/lib/test/i18n.test.js @@ -0,0 +1,41 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +var assert = require("assert"); +var i18n = require("../i18n"); +suite('XLF Parser Tests', function () { + var sampleXlf = 'Key #1Key #2 &'; + var sampleTranslatedXlf = 'Key #1Кнопка #1Key #2 &Кнопка #2 &'; + var originalFilePath = 'vs/base/common/keybinding'; + var keys = ['key1', 'key2']; + var messages = ['Key #1', 'Key #2 &']; + var translatedMessages = { key1: 'Кнопка #1', key2: 'Кнопка #2 &' }; + test('Keys & messages to XLF conversion', function () { + var xlf = new i18n.XLF('vscode-workbench'); + xlf.addFile(originalFilePath, keys, messages); + var xlfString = xlf.toString(); + assert.strictEqual(xlfString.replace(/\s{2,}/g, ''), sampleXlf); + }); + test('XLF to keys & messages conversion', function () { + i18n.XLF.parse(sampleTranslatedXlf).then(function (resolvedFiles) { + assert.deepEqual(resolvedFiles[0].messages, translatedMessages); + assert.strictEqual(resolvedFiles[0].originalFilePath, originalFilePath); + }); + }); + test('JSON file source path to Transifex resource match', function () { + var editorProject = 'vscode-editor', workbenchProject = 'vscode-workbench'; + var platform = { name: 'vs/platform', project: editorProject }, editorContrib = { name: 'vs/editor/contrib', project: editorProject }, editor = { name: 'vs/editor', project: editorProject }, base = { name: 'vs/base', project: editorProject }, code = { name: 'vs/code', project: workbenchProject }, workbenchParts = { name: 'vs/workbench/parts/html', project: workbenchProject }, workbenchServices = { name: 'vs/workbench/services/files', project: workbenchProject }, workbench = { name: 'vs/workbench', project: workbenchProject }; + assert.deepEqual(i18n.getResource('vs/platform/actions/browser/menusExtensionPoint'), platform); + assert.deepEqual(i18n.getResource('vs/editor/contrib/clipboard/browser/clipboard'), editorContrib); + assert.deepEqual(i18n.getResource('vs/editor/common/modes/modesRegistry'), editor); + assert.deepEqual(i18n.getResource('vs/base/common/errorMessage'), base); + assert.deepEqual(i18n.getResource('vs/code/electron-main/window'), code); + assert.deepEqual(i18n.getResource('vs/workbench/parts/html/browser/webview'), workbenchParts); + assert.deepEqual(i18n.getResource('vs/workbench/services/files/node/fileService'), workbenchServices); + assert.deepEqual(i18n.getResource('vs/workbench/browser/parts/panel/panelActions'), workbench); + }); +}); +//# sourceMappingURL=i18n.test.js.map \ No newline at end of file diff --git a/build/lib/test/i18n.test.ts b/build/lib/test/i18n.test.ts new file mode 100644 index 00000000000..bf85d091b08 --- /dev/null +++ b/build/lib/test/i18n.test.ts @@ -0,0 +1,54 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert = require('assert'); +import i18n = require('../i18n'); + +suite('XLF Parser Tests', () => { + const sampleXlf = 'Key #1Key #2 &'; + const sampleTranslatedXlf = 'Key #1Кнопка #1Key #2 &Кнопка #2 &'; + const originalFilePath = 'vs/base/common/keybinding'; + const keys = ['key1', 'key2']; + const messages = ['Key #1', 'Key #2 &']; + const translatedMessages = { key1: 'Кнопка #1', key2: 'Кнопка #2 &' }; + + test('Keys & messages to XLF conversion', () => { + let xlf = new i18n.XLF('vscode-workbench'); + xlf.addFile(originalFilePath, keys, messages); + const xlfString = xlf.toString(); + + assert.strictEqual(xlfString.replace(/\s{2,}/g, ''), sampleXlf); + }); + + test('XLF to keys & messages conversion', () => { + i18n.XLF.parse(sampleTranslatedXlf).then(function(resolvedFiles) { + assert.deepEqual(resolvedFiles[0].messages, translatedMessages); + assert.strictEqual(resolvedFiles[0].originalFilePath, originalFilePath); + }); + }); + + test('JSON file source path to Transifex resource match', () => { + const editorProject: string = 'vscode-editor', + workbenchProject: string = 'vscode-workbench'; + + const platform: i18n.Resource = { name: 'vs/platform', project: editorProject }, + editorContrib = { name: 'vs/editor/contrib', project: editorProject }, + editor = { name: 'vs/editor', project: editorProject }, + base = { name: 'vs/base', project: editorProject }, + code = { name: 'vs/code', project: workbenchProject }, + workbenchParts = { name: 'vs/workbench/parts/html', project: workbenchProject }, + workbenchServices = { name: 'vs/workbench/services/files', project: workbenchProject }, + workbench = { name: 'vs/workbench', project: workbenchProject}; + + assert.deepEqual(i18n.getResource('vs/platform/actions/browser/menusExtensionPoint'), platform); + assert.deepEqual(i18n.getResource('vs/editor/contrib/clipboard/browser/clipboard'), editorContrib); + assert.deepEqual(i18n.getResource('vs/editor/common/modes/modesRegistry'), editor); + assert.deepEqual(i18n.getResource('vs/base/common/errorMessage'), base); + assert.deepEqual(i18n.getResource('vs/code/electron-main/window'), code); + assert.deepEqual(i18n.getResource('vs/workbench/parts/html/browser/webview'), workbenchParts); + assert.deepEqual(i18n.getResource('vs/workbench/services/files/node/fileService'), workbenchServices); + assert.deepEqual(i18n.getResource('vs/workbench/browser/parts/panel/panelActions'), workbench); + }); +}); \ No newline at end of file From bf008106c8d4d103b868d7483cff4cf133f9c673 Mon Sep 17 00:00:00 2001 From: Michel Kaporin Date: Fri, 17 Mar 2017 17:24:18 +0100 Subject: [PATCH 4/5] Synchronised pipe to prevent its ending before all of the API responses received. --- build/gulpfile.vscode.js | 27 +++------ build/lib/i18n.js | 109 ++++++++++++++++++++--------------- build/lib/i18n.ts | 110 +++++++++++++++++++++--------------- build/lib/test/i18n.test.js | 1 - 4 files changed, 138 insertions(+), 109 deletions(-) diff --git a/build/gulpfile.vscode.js b/build/gulpfile.vscode.js index 21432d1a217..160e46abbf4 100644 --- a/build/gulpfile.vscode.js +++ b/build/gulpfile.vscode.js @@ -349,27 +349,18 @@ gulp.task('vscode-translations-update', function() { const pathToExtensions = './extensions/**/*.nls.json'; const pathToSetup = 'build/win32/**/{Default.isl,messages.en.isl}'; - gulp.src(pathToMetadata) - .pipe(i18n.prepareXlfFiles()) - .pipe(i18n.pushXlfFiles(apiUrl, apiName, apiToken)); - - gulp.src(pathToSetup) - .pipe(i18n.prepareXlfFiles()) - .pipe(i18n.pushXlfFiles(apiUrl, apiName, apiToken)); - - return gulp.src(pathToExtensions) - .pipe(i18n.prepareXlfFiles('vscode-extensions')) - .pipe(i18n.pushXlfFiles(apiUrl, apiName, apiToken)); + return es.merge( + gulp.src(pathToMetadata).pipe(i18n.prepareXlfFiles()), + gulp.src(pathToSetup).pipe(i18n.prepareXlfFiles()), + gulp.src(pathToExtensions).pipe(i18n.prepareXlfFiles('vscode-extensions')) + ).pipe(i18n.pushXlfFiles(apiUrl, apiName, apiToken)); }); gulp.task('vscode-translations-pull', function() { - i18n.pullXlfFiles('vscode-editor-workbench', apiUrl, apiName, apiToken) - .pipe(i18n.prepareJsonFiles()) - .pipe(vfs.dest('./i18n')); - - return i18n.pullXlfFiles('vscode-extensions', apiUrl, apiName, apiToken) - .pipe(i18n.prepareJsonFiles()) - .pipe(vfs.dest('./i18n')); + return es.merge( + i18n.pullXlfFiles('vscode-editor-workbench', apiUrl, apiName, apiToken), + i18n.pullXlfFiles('vscode-extensions', apiUrl, apiName, apiToken) + ).pipe(i18n.prepareJsonFiles()).pipe(vfs.dest('C:/Users/t-mikapo/Documents/Contribution/Localisation/transifex_export/vscode')); // './i18n' }); // Sourcemaps diff --git a/build/lib/i18n.js b/build/lib/i18n.js index 5d00caf57fa..4a16a55fbf7 100644 --- a/build/lib/i18n.js +++ b/build/lib/i18n.js @@ -535,7 +535,7 @@ function getResource(sourceFile) { } exports.getResource = getResource; function importBundleJson(file, json, stream) { - var transifexEditorXlfs = Object.create(null); + var bundleXlfs = Object.create(null); for (var source in json.keys) { var projectResource = getResource(source); var resource = projectResource.name; @@ -545,12 +545,12 @@ function importBundleJson(file, json, stream) { if (keys.length !== messages.length) { log('Error:', "There is a mismatch between keys and messages in " + file.relative); } - var xlf = transifexEditorXlfs[resource] ? transifexEditorXlfs[resource] : transifexEditorXlfs[resource] = new XLF(project); + var xlf = bundleXlfs[resource] ? bundleXlfs[resource] : bundleXlfs[resource] = new XLF(project); xlf.addFile(source, keys, messages); } - for (var resource in transifexEditorXlfs) { - var newFilePath = transifexEditorXlfs[resource].project + "/" + resource.replace(/\//g, '_') + ".xlf"; - var xlfFile = new File({ path: newFilePath, contents: new Buffer(transifexEditorXlfs[resource].toString(), 'utf-8') }); + for (var resource in bundleXlfs) { + var newFilePath = bundleXlfs[resource].project + "/" + resource.replace(/\//g, '_') + ".xlf"; + var xlfFile = new File({ path: newFilePath, contents: new Buffer(bundleXlfs[resource].toString(), 'utf-8') }); stream.emit('data', xlfFile); } } @@ -637,6 +637,8 @@ function importIsl(file, stream) { } } function pushXlfFiles(apiUrl, username, password) { + var tryGetPromises = []; + var updateCreatePromises = []; return event_stream_1.through(function (file) { var project = path.dirname(file.relative); var fileName = path.basename(file.path); @@ -646,13 +648,24 @@ function pushXlfFiles(apiUrl, username, password) { 'password': password }; // Check if resource already exists, if not, then create it. - tryGetResource(project, slug, apiUrl, credentials).then(function (exists) { + var promise = tryGetResource(project, slug, apiUrl, credentials); + tryGetPromises.push(promise); + promise.then(function (exists) { if (exists) { - updateResource(project, slug, file, apiUrl, credentials); + promise = updateResource(project, slug, file, apiUrl, credentials); } else { - createResource(project, slug, file, apiUrl, credentials); + promise = createResource(project, slug, file, apiUrl, credentials); } + updateCreatePromises.push(promise); + }); + }, function () { + var _this = this; + // End the pipe only after all the communication with Transifex API happened + Promise.all(tryGetPromises).then(function () { + Promise.all(updateCreatePromises).then(function () { + _this.emit('end'); + }); }); }); } @@ -674,27 +687,30 @@ function tryGetResource(project, slug, apiUrl, credentials) { }); } function createResource(project, slug, xlfFile, apiUrl, credentials) { - var url = apiUrl + "/project/" + project + "/resources"; - var options = { - 'body': { - 'content': xlfFile.contents.toString(), - 'name': slug, - 'slug': slug, - 'i18n_type': 'XLIFF' - }, - 'json': true, - 'auth': credentials - }; - request.post(url, options, function (err, res) { - if (err) { - log('Error:', "Failed to create Transifex " + project + "/" + slug + ": " + err); - } - if (res.statusCode === 201) { - log("Resource " + project + "/" + slug + " successfully created on Transifex."); - } - else { - log('Error:', "Something went wrong creating " + slug + " in " + project + ". " + res.statusCode); - } + return new Promise(function (resolve) { + var url = apiUrl + "/project/" + project + "/resources"; + var options = { + 'body': { + 'content': xlfFile.contents.toString(), + 'name': slug, + 'slug': slug, + 'i18n_type': 'XLIFF' + }, + 'json': true, + 'auth': credentials + }; + request.post(url, options, function (err, res) { + if (err) { + log('Error:', "Failed to create Transifex " + project + "/" + slug + ": " + err); + } + if (res.statusCode === 201) { + log("Resource " + project + "/" + slug + " successfully created on Transifex."); + resolve(); + } + else { + log('Error:', "Something went wrong creating " + slug + " in " + project + ". " + res.statusCode); + } + }); }); } /** @@ -702,22 +718,25 @@ function createResource(project, slug, xlfFile, apiUrl, credentials) { * https://dev.befoolish.co/tx-docs/public/projects/updating-content#what-happens-when-you-update-files */ function updateResource(project, slug, xlfFile, apiUrl, credentials) { - var url = apiUrl + "/project/" + project + "/resource/" + slug + "/content"; - var options = { - 'body': { 'content': xlfFile.contents.toString() }, - 'json': true, - 'auth': credentials - }; - request.put(url, options, function (err, res, body) { - if (err) { - log('Error:', "Failed to update Transifex " + project + "/" + slug + ": " + err); - } - if (res.statusCode === 200) { - log("Resource " + project + "/" + slug + " successfully updated on Transifex. Strings added: " + body['strings_added'] + ", updated: " + body['strings_updated'] + ", deleted: " + body['strings_delete']); - } - else { - log('Error:', "Something went wrong updating " + slug + " in " + project + ". " + res.statusCode); - } + return new Promise(function (resolve) { + var url = apiUrl + "/project/" + project + "/resource/" + slug + "/content"; + var options = { + 'body': { 'content': xlfFile.contents.toString() }, + 'json': true, + 'auth': credentials + }; + request.put(url, options, function (err, res, body) { + if (err) { + log('Error:', "Failed to update Transifex " + project + "/" + slug + ": " + err); + } + if (res.statusCode === 200) { + log("Resource " + project + "/" + slug + " successfully updated on Transifex. Strings added: " + body['strings_added'] + ", updated: " + body['strings_updated'] + ", deleted: " + body['strings_delete']); + resolve(); + } + else { + log('Error:', "Something went wrong updating " + slug + " in " + project + ". " + res.statusCode); + } + }); }); } function getMetadataResources(pathToMetadata) { diff --git a/build/lib/i18n.ts b/build/lib/i18n.ts index 9c5f1b89698..10708e23705 100644 --- a/build/lib/i18n.ts +++ b/build/lib/i18n.ts @@ -600,7 +600,7 @@ export function getResource(sourceFile: string): Resource { function importBundleJson(file: File, json: BundledFormat, stream: ThroughStream): void { - let transifexEditorXlfs: Map = Object.create(null); + let bundleXlfs: Map = Object.create(null); for (let source in json.keys) { const projectResource = getResource(source); @@ -613,13 +613,13 @@ function importBundleJson(file: File, json: BundledFormat, stream: ThroughStream log('Error:', `There is a mismatch between keys and messages in ${file.relative}`); } - let xlf = transifexEditorXlfs[resource] ? transifexEditorXlfs[resource] : transifexEditorXlfs[resource] = new XLF(project); + let xlf = bundleXlfs[resource] ? bundleXlfs[resource] : bundleXlfs[resource] = new XLF(project); xlf.addFile(source, keys, messages); } - for (let resource in transifexEditorXlfs) { - const newFilePath = `${transifexEditorXlfs[resource].project}/${resource.replace(/\//g, '_')}.xlf`; - const xlfFile = new File({ path: newFilePath, contents: new Buffer(transifexEditorXlfs[resource].toString(), 'utf-8')}); + for (let resource in bundleXlfs) { + const newFilePath = `${bundleXlfs[resource].project}/${resource.replace(/\//g, '_')}.xlf`; + const xlfFile = new File({ path: newFilePath, contents: new Buffer(bundleXlfs[resource].toString(), 'utf-8')}); stream.emit('data', xlfFile); } } @@ -720,6 +720,9 @@ function importIsl(file: File, stream: ThroughStream) { } export function pushXlfFiles(apiUrl: string, username: string, password: string): ThroughStream { + let tryGetPromises = []; + let updateCreatePromises = []; + return through(function(file: File) { const project = path.dirname(file.relative); const fileName = path.basename(file.path); @@ -730,12 +733,23 @@ export function pushXlfFiles(apiUrl: string, username: string, password: string) }; // Check if resource already exists, if not, then create it. - tryGetResource(project, slug, apiUrl, credentials).then(exists => { + let promise = tryGetResource(project, slug, apiUrl, credentials); + tryGetPromises.push(promise); + promise.then(exists => { if (exists) { - updateResource(project, slug, file, apiUrl, credentials); + promise = updateResource(project, slug, file, apiUrl, credentials); } else { - createResource(project, slug, file, apiUrl, credentials); + promise = createResource(project, slug, file, apiUrl, credentials); } + updateCreatePromises.push(promise); + }); + + }, function() { + // End the pipe only after all the communication with Transifex API happened + Promise.all(tryGetPromises).then(() => { + Promise.all(updateCreatePromises).then(() => { + this.emit('end'); + }); }); }); } @@ -755,29 +769,32 @@ function tryGetResource(project: string, slug: string, apiUrl: string, credentia }); } -function createResource(project: string, slug: string, xlfFile: File, apiUrl:string, credentials: any): void { - const url = `${apiUrl}/project/${project}/resources`; - const options = { - 'body': { - 'content': xlfFile.contents.toString(), - 'name': slug, - 'slug': slug, - 'i18n_type': 'XLIFF' - }, - 'json': true, - 'auth': credentials - }; +function createResource(project: string, slug: string, xlfFile: File, apiUrl:string, credentials: any): Promise { + return new Promise((resolve) => { + const url = `${apiUrl}/project/${project}/resources`; + const options = { + 'body': { + 'content': xlfFile.contents.toString(), + 'name': slug, + 'slug': slug, + 'i18n_type': 'XLIFF' + }, + 'json': true, + 'auth': credentials + }; - request.post(url, options, function(err, res) { - if (err) { - log('Error:', `Failed to create Transifex ${project}/${slug}: ${err}`); - } + request.post(url, options, function(err, res) { + if (err) { + log('Error:', `Failed to create Transifex ${project}/${slug}: ${err}`); + } - if (res.statusCode === 201) { - log(`Resource ${project}/${slug} successfully created on Transifex.`); - } else { - log('Error:', `Something went wrong creating ${slug} in ${project}. ${res.statusCode}`); - } + if (res.statusCode === 201) { + log(`Resource ${project}/${slug} successfully created on Transifex.`); + resolve(); + } else { + log('Error:', `Something went wrong creating ${slug} in ${project}. ${res.statusCode}`); + } + }); }); } @@ -785,24 +802,27 @@ function createResource(project: string, slug: string, xlfFile: File, apiUrl:str * The following link provides information about how Transifex handles updates of a resource file: * https://dev.befoolish.co/tx-docs/public/projects/updating-content#what-happens-when-you-update-files */ -function updateResource(project: string, slug: string, xlfFile: File, apiUrl: string, credentials: any) : void { - const url = `${apiUrl}/project/${project}/resource/${slug}/content`; - const options = { - 'body': { 'content': xlfFile.contents.toString() }, - 'json': true, - 'auth': credentials - }; +function updateResource(project: string, slug: string, xlfFile: File, apiUrl: string, credentials: any) : Promise { + return new Promise((resolve) => { + const url = `${apiUrl}/project/${project}/resource/${slug}/content`; + const options = { + 'body': { 'content': xlfFile.contents.toString() }, + 'json': true, + 'auth': credentials + }; - request.put(url, options, function(err, res, body) { - if (err) { - log('Error:', `Failed to update Transifex ${project}/${slug}: ${err}`); - } + request.put(url, options, function(err, res, body) { + if (err) { + log('Error:', `Failed to update Transifex ${project}/${slug}: ${err}`); + } - if (res.statusCode === 200) { - log(`Resource ${project}/${slug} successfully updated on Transifex. Strings added: ${body['strings_added']}, updated: ${body['strings_updated']}, deleted: ${body['strings_delete']}`); - } else { - log('Error:', `Something went wrong updating ${slug} in ${project}. ${res.statusCode}`); - } + if (res.statusCode === 200) { + log(`Resource ${project}/${slug} successfully updated on Transifex. Strings added: ${body['strings_added']}, updated: ${body['strings_updated']}, deleted: ${body['strings_delete']}`); + resolve(); + } else { + log('Error:', `Something went wrong updating ${slug} in ${project}. ${res.statusCode}`); + } + }); }); } diff --git a/build/lib/test/i18n.test.js b/build/lib/test/i18n.test.js index 4e7b60b459e..03daa50a66c 100644 --- a/build/lib/test/i18n.test.js +++ b/build/lib/test/i18n.test.js @@ -38,4 +38,3 @@ suite('XLF Parser Tests', function () { assert.deepEqual(i18n.getResource('vs/workbench/browser/parts/panel/panelActions'), workbench); }); }); -//# sourceMappingURL=i18n.test.js.map \ No newline at end of file From 980ea6b2c26b3055834726f1a4413831aa8d27bd Mon Sep 17 00:00:00 2001 From: Michel Kaporin Date: Mon, 20 Mar 2017 15:01:18 +0100 Subject: [PATCH 5/5] Updated imports for i18n with typings. Removed 'request' module dependency. --- build/lib/i18n.js | 165 +++++++++++++++----------- build/lib/i18n.ts | 176 ++++++++++++++++------------ build/lib/typings/event-stream.d.ts | 2 + build/lib/typings/through.d.ts | 22 ---- build/lib/typings/vinyl.d.ts | 112 ------------------ package.json | 6 +- 6 files changed, 202 insertions(+), 281 deletions(-) delete mode 100644 build/lib/typings/through.d.ts delete mode 100644 build/lib/typings/vinyl.d.ts diff --git a/build/lib/i18n.js b/build/lib/i18n.js index 4a16a55fbf7..1f8f5a8e0d6 100644 --- a/build/lib/i18n.js +++ b/build/lib/i18n.js @@ -11,9 +11,8 @@ var File = require("vinyl"); var Is = require("is"); var xml2js = require("xml2js"); var glob = require("glob"); +var http = require("http"); var util = require('gulp-util'); -var request = require('request'); -var es = require('event-stream'); var iconv = require('iconv-lite'); function log(message) { var rest = []; @@ -636,28 +635,27 @@ function importIsl(file, stream) { stream.emit('data', xlfFile); } } -function pushXlfFiles(apiUrl, username, password) { +function pushXlfFiles(apiHostname, username, password) { var tryGetPromises = []; var updateCreatePromises = []; return event_stream_1.through(function (file) { var project = path.dirname(file.relative); var fileName = path.basename(file.path); var slug = fileName.substr(0, fileName.length - '.xlf'.length); - var credentials = { - 'user': username, - 'password': password - }; + var credentials = username + ":" + password; // Check if resource already exists, if not, then create it. - var promise = tryGetResource(project, slug, apiUrl, credentials); + var promise = tryGetResource(project, slug, apiHostname, credentials); tryGetPromises.push(promise); promise.then(function (exists) { if (exists) { - promise = updateResource(project, slug, file, apiUrl, credentials); + promise = updateResource(project, slug, file, apiHostname, credentials); } else { - promise = createResource(project, slug, file, apiUrl, credentials); + promise = createResource(project, slug, file, apiHostname, credentials); } updateCreatePromises.push(promise); + }).catch(function (reason) { + log('Error:', reason); }); }, function () { var _this = this; @@ -665,15 +663,20 @@ function pushXlfFiles(apiUrl, username, password) { Promise.all(tryGetPromises).then(function () { Promise.all(updateCreatePromises).then(function () { _this.emit('end'); - }); - }); + }).catch(function (reason) { return log('Error:', reason); }); + }).catch(function (reason) { return log('Error:', reason); }); }); } exports.pushXlfFiles = pushXlfFiles; -function tryGetResource(project, slug, apiUrl, credentials) { +function tryGetResource(project, slug, apiHostname, credentials) { return new Promise(function (resolve, reject) { - var url = apiUrl + "/project/" + project + "/resource/" + slug + "/?details"; - request.get(url, { 'auth': credentials }).on('response', function (response) { + var options = { + hostname: apiHostname, + path: "/api/2/project/" + project + "/resource/" + slug + "/?details", + auth: credentials, + method: 'GET' + }; + var request = http.request(options, function (response) { if (response.statusCode === 404) { resolve(false); } @@ -681,62 +684,84 @@ function tryGetResource(project, slug, apiUrl, credentials) { resolve(true); } else { - reject("Failed to query resource " + slug + ". Response: " + response.statusCode + " " + response.statusMessage); + reject("Failed to query resource " + project + "/" + slug + ". Response: " + response.statusCode + " " + response.statusMessage); } + }).on('error', function (err) { + reject("Failed to get " + project + "/" + slug + " on Transifex: " + err); }); + request.end(); }); } -function createResource(project, slug, xlfFile, apiUrl, credentials) { - return new Promise(function (resolve) { - var url = apiUrl + "/project/" + project + "/resources"; +function createResource(project, slug, xlfFile, apiHostname, credentials) { + return new Promise(function (resolve, reject) { + var data = JSON.stringify({ + 'content': xlfFile.contents.toString(), + 'name': slug, + 'slug': slug, + 'i18n_type': 'XLIFF' + }); var options = { - 'body': { - 'content': xlfFile.contents.toString(), - 'name': slug, - 'slug': slug, - 'i18n_type': 'XLIFF' + hostname: apiHostname, + path: "/api/2/project/" + project + "/resources", + headers: { + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(data) }, - 'json': true, - 'auth': credentials + auth: credentials, + method: 'POST' }; - request.post(url, options, function (err, res) { - if (err) { - log('Error:', "Failed to create Transifex " + project + "/" + slug + ": " + err); - } + var request = http.request(options, function (res) { if (res.statusCode === 201) { log("Resource " + project + "/" + slug + " successfully created on Transifex."); - resolve(); } else { - log('Error:', "Something went wrong creating " + slug + " in " + project + ". " + res.statusCode); + reject("Something went wrong in the request creating " + slug + " in " + project + ". " + res.statusCode); } + }).on('error', function (err) { + reject("Failed to create " + project + "/" + slug + " on Transifex: " + err); }); + request.write(data); + request.end(); }); } /** * The following link provides information about how Transifex handles updates of a resource file: * https://dev.befoolish.co/tx-docs/public/projects/updating-content#what-happens-when-you-update-files */ -function updateResource(project, slug, xlfFile, apiUrl, credentials) { - return new Promise(function (resolve) { - var url = apiUrl + "/project/" + project + "/resource/" + slug + "/content"; +function updateResource(project, slug, xlfFile, apiHostname, credentials) { + return new Promise(function (resolve, reject) { + var data = JSON.stringify({ content: xlfFile.contents.toString() }); var options = { - 'body': { 'content': xlfFile.contents.toString() }, - 'json': true, - 'auth': credentials + hostname: apiHostname, + path: "/api/2/project/" + project + "/resource/" + slug + "/content", + headers: { + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(data) + }, + auth: credentials, + method: 'PUT' }; - request.put(url, options, function (err, res, body) { - if (err) { - log('Error:', "Failed to update Transifex " + project + "/" + slug + ": " + err); - } + var request = http.request(options, function (res) { if (res.statusCode === 200) { - log("Resource " + project + "/" + slug + " successfully updated on Transifex. Strings added: " + body['strings_added'] + ", updated: " + body['strings_updated'] + ", deleted: " + body['strings_delete']); - resolve(); + res.setEncoding('utf8'); + var responseBuffer_1 = ''; + res.on('data', function (chunk) { + responseBuffer_1 += chunk; + }); + res.on('end', function () { + var response = JSON.parse(responseBuffer_1); + log("Resource " + project + "/" + slug + " successfully updated on Transifex. Strings added: " + response.strings_added + ", updated: " + response.strings_added + ", deleted: " + response.strings_added); + resolve(); + }); } else { - log('Error:', "Something went wrong updating " + slug + " in " + project + ". " + res.statusCode); + reject("Something went wrong in the request updating " + slug + " in " + project + ". " + res.statusCode); } + }).on('error', function (err) { + reject("Failed to update " + project + "/" + slug + " on Transifex: " + err); }); + request.write(data); + request.end(); }); } function getMetadataResources(pathToMetadata) { @@ -773,22 +798,17 @@ function obtainProjectResources(projectName) { } return resources; } -function pullXlfFiles(projectName, apiUrl, username, password, resources) { +function pullXlfFiles(projectName, apiHostname, username, password, resources) { if (!resources) { resources = obtainProjectResources(projectName); } if (!resources) { throw new Error('Transifex projects and resources must be defined to be able to pull translations from Transifex.'); } - var credentials = { - 'auth': { - 'user': username, - 'password': password - } - }; + var credentials = username + ":" + password; var expectedTranslationsCount = vscodeLanguages.length * resources.length; var translationsRetrieved = 0, called = false; - return es.readable(function (count, callback) { + return event_stream_1.readable(function (count, callback) { // Mark end of stream when all resources were retrieved if (translationsRetrieved === expectedTranslationsCount) { return this.emit('end'); @@ -801,25 +821,28 @@ function pullXlfFiles(projectName, apiUrl, username, password, resources) { var slug = resource.name.replace(/\//g, '_'); var project = resource.project; var iso639 = iso639_3_to_2[language]; - var url = apiUrl + "/project/" + project + "/resource/" + slug + "/translation/" + iso639 + "?file&mode=onlyreviewed"; - var xlfBuffer = '', responseCode; - request.get(url, credentials) - .on('response', function (response) { - responseCode = response.statusCode; - }) - .on('data', function (data) { return xlfBuffer += data; }) - .on('end', function () { - if (responseCode === 200) { - stream_1.emit('data', new File({ contents: new Buffer(xlfBuffer) })); - } - else { - log('Error:', slug + " in " + project + " returned no data. Response code: " + responseCode + "."); - } - translationsRetrieved++; - }) - .on('error', function (error) { - log('Error:', "Failed to query resource " + slug + " with the following error: " + error); + var options = { + hostname: apiHostname, + path: "/api/2/project/" + project + "/resource/" + slug + "/translation/" + iso639 + "?file&mode=onlyreviewed", + auth: credentials, + method: 'GET' + }; + var request = http.request(options, function (res) { + var xlfBuffer = ''; + res.on('data', function (data) { return xlfBuffer += data; }); + res.on('end', function () { + if (res.statusCode === 200) { + stream_1.emit('data', new File({ contents: new Buffer(xlfBuffer) })); + } + else { + log('Error:', slug + " in " + project + " returned no data. Response code: " + res.statusCode + "."); + } + translationsRetrieved++; + }); + }).on('error', function (err) { + log('Error:', "Failed to query resource " + slug + " with the following error: " + err); }); + request.end(); }); }); } diff --git a/build/lib/i18n.ts b/build/lib/i18n.ts index 10708e23705..24971e98500 100644 --- a/build/lib/i18n.ts +++ b/build/lib/i18n.ts @@ -6,16 +6,15 @@ import * as path from 'path'; import * as fs from 'fs'; -import { through } from 'event-stream'; +import { through, readable } from 'event-stream'; import { ThroughStream } from 'through'; import File = require('vinyl'); import * as Is from 'is'; -import xml2js = require('xml2js'); +import * as xml2js from 'xml2js'; import * as glob from 'glob'; +import * as http from 'http'; var util = require('gulp-util'); -const request = require('request'); -const es = require('event-stream'); var iconv = require('iconv-lite'); function log(message: any, ...rest: any[]): void { @@ -719,7 +718,7 @@ function importIsl(file: File, stream: ThroughStream) { } } -export function pushXlfFiles(apiUrl: string, username: string, password: string): ThroughStream { +export function pushXlfFiles(apiHostname: string, username: string, password: string): ThroughStream { let tryGetPromises = []; let updateCreatePromises = []; @@ -727,21 +726,20 @@ export function pushXlfFiles(apiUrl: string, username: string, password: string) const project = path.dirname(file.relative); const fileName = path.basename(file.path); const slug = fileName.substr(0, fileName.length - '.xlf'.length); - const credentials = { - 'user': username, - 'password': password - }; + const credentials = `${username}:${password}`; // Check if resource already exists, if not, then create it. - let promise = tryGetResource(project, slug, apiUrl, credentials); + let promise = tryGetResource(project, slug, apiHostname, credentials); tryGetPromises.push(promise); promise.then(exists => { if (exists) { - promise = updateResource(project, slug, file, apiUrl, credentials); + promise = updateResource(project, slug, file, apiHostname, credentials); } else { - promise = createResource(project, slug, file, apiUrl, credentials); + promise = createResource(project, slug, file, apiHostname, credentials); } updateCreatePromises.push(promise); + }).catch((reason) => { + log('Error:', reason); }); }, function() { @@ -749,52 +747,67 @@ export function pushXlfFiles(apiUrl: string, username: string, password: string) Promise.all(tryGetPromises).then(() => { Promise.all(updateCreatePromises).then(() => { this.emit('end'); - }); - }); + }).catch((reason) => log('Error:', reason)); + }).catch((reason) => log('Error:', reason)); }); } -function tryGetResource(project: string, slug: string, apiUrl: string, credentials: any): Promise { +function tryGetResource(project: string, slug: string, apiHostname: string, credentials: string): Promise { return new Promise((resolve, reject) => { - const url = `${apiUrl}/project/${project}/resource/${slug}/?details`; - request.get(url, { 'auth': credentials }).on('response', function (response) { + const options = { + hostname: apiHostname, + path: `/api/2/project/${project}/resource/${slug}/?details`, + auth: credentials, + method: 'GET' + }; + + const request = http.request(options, (response) => { if (response.statusCode === 404) { resolve(false); } else if (response.statusCode === 200) { resolve(true); } else { - reject(`Failed to query resource ${slug}. Response: ${response.statusCode} ${response.statusMessage}`); + reject(`Failed to query resource ${project}/${slug}. Response: ${response.statusCode} ${response.statusMessage}`); } + }).on('error', (err) => { + reject(`Failed to get ${project}/${slug} on Transifex: ${err}`); }); + + request.end(); }); } -function createResource(project: string, slug: string, xlfFile: File, apiUrl:string, credentials: any): Promise { - return new Promise((resolve) => { - const url = `${apiUrl}/project/${project}/resources`; +function createResource(project: string, slug: string, xlfFile: File, apiHostname: string, credentials: any): Promise { + return new Promise((resolve, reject) => { + const data = JSON.stringify({ + 'content': xlfFile.contents.toString(), + 'name': slug, + 'slug': slug, + 'i18n_type': 'XLIFF' + }); const options = { - 'body': { - 'content': xlfFile.contents.toString(), - 'name': slug, - 'slug': slug, - 'i18n_type': 'XLIFF' + hostname: apiHostname, + path: `/api/2/project/${project}/resources`, + headers: { + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(data) }, - 'json': true, - 'auth': credentials + auth: credentials, + method: 'POST' }; - request.post(url, options, function(err, res) { - if (err) { - log('Error:', `Failed to create Transifex ${project}/${slug}: ${err}`); - } - + let request = http.request(options, (res) => { if (res.statusCode === 201) { log(`Resource ${project}/${slug} successfully created on Transifex.`); - resolve(); } else { - log('Error:', `Something went wrong creating ${slug} in ${project}. ${res.statusCode}`); + reject(`Something went wrong in the request creating ${slug} in ${project}. ${res.statusCode}`); } + }).on('error', (err) => { + reject(`Failed to create ${project}/${slug} on Transifex: ${err}`); }); + + request.write(data); + request.end(); }); } @@ -802,27 +815,42 @@ function createResource(project: string, slug: string, xlfFile: File, apiUrl:str * The following link provides information about how Transifex handles updates of a resource file: * https://dev.befoolish.co/tx-docs/public/projects/updating-content#what-happens-when-you-update-files */ -function updateResource(project: string, slug: string, xlfFile: File, apiUrl: string, credentials: any) : Promise { - return new Promise((resolve) => { - const url = `${apiUrl}/project/${project}/resource/${slug}/content`; +function updateResource(project: string, slug: string, xlfFile: File, apiHostname: string, credentials: string) : Promise { + return new Promise((resolve, reject) => { + const data = JSON.stringify({ content: xlfFile.contents.toString() }); const options = { - 'body': { 'content': xlfFile.contents.toString() }, - 'json': true, - 'auth': credentials + hostname: apiHostname, + path: `/api/2/project/${project}/resource/${slug}/content`, + headers: { + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(data) + }, + auth: credentials, + method: 'PUT' }; - request.put(url, options, function(err, res, body) { - if (err) { - log('Error:', `Failed to update Transifex ${project}/${slug}: ${err}`); - } - + let request = http.request(options, (res) => { if (res.statusCode === 200) { - log(`Resource ${project}/${slug} successfully updated on Transifex. Strings added: ${body['strings_added']}, updated: ${body['strings_updated']}, deleted: ${body['strings_delete']}`); - resolve(); + res.setEncoding('utf8'); + + let responseBuffer: string = ''; + res.on('data', function (chunk) { + responseBuffer += chunk; + }); + res.on('end', () => { + const response = JSON.parse(responseBuffer); + log(`Resource ${project}/${slug} successfully updated on Transifex. Strings added: ${response.strings_added}, updated: ${response.strings_added}, deleted: ${response.strings_added}`); + resolve(); + }); } else { - log('Error:', `Something went wrong updating ${slug} in ${project}. ${res.statusCode}`); + reject(`Something went wrong in the request updating ${slug} in ${project}. ${res.statusCode}`); } + }).on('error', (err) => { + reject(`Failed to update ${project}/${slug} on Transifex: ${err}`); }); + + request.write(data); + request.end(); }); } @@ -863,7 +891,7 @@ function obtainProjectResources(projectName: string): Resource[] { return resources; } -export function pullXlfFiles(projectName: string, apiUrl: string, username: string, password: string, resources?: Resource[]): NodeJS.ReadableStream { +export function pullXlfFiles(projectName: string, apiHostname: string, username: string, password: string, resources?: Resource[]): NodeJS.ReadableStream { if (!resources) { resources = obtainProjectResources(projectName); } @@ -871,16 +899,11 @@ export function pullXlfFiles(projectName: string, apiUrl: string, username: stri throw new Error('Transifex projects and resources must be defined to be able to pull translations from Transifex.'); } - const credentials = { - 'auth': { - 'user': username, - 'password': password - } - }; + const credentials = `${username}:${password}`; let expectedTranslationsCount = vscodeLanguages.length * resources.length; let translationsRetrieved = 0, called = false; - return es.readable(function(count, callback) { + return readable(function(count, callback) { // Mark end of stream when all resources were retrieved if (translationsRetrieved === expectedTranslationsCount) { return this.emit('end'); @@ -895,25 +918,28 @@ export function pullXlfFiles(projectName: string, apiUrl: string, username: stri const slug = resource.name.replace(/\//g, '_'); const project = resource.project; const iso639 = iso639_3_to_2[language]; - const url = `${apiUrl}/project/${project}/resource/${slug}/translation/${iso639}?file&mode=onlyreviewed`; + const options = { + hostname: apiHostname, + path: `/api/2/project/${project}/resource/${slug}/translation/${iso639}?file&mode=onlyreviewed`, + auth: credentials, + method: 'GET' + }; - let xlfBuffer: string = '', responseCode: number; - request.get(url, credentials) - .on('response', (response) => { - responseCode = response.statusCode; - }) - .on('data', (data) => xlfBuffer += data) - .on('end', () => { - if (responseCode === 200) { - stream.emit('data', new File({ contents: new Buffer(xlfBuffer) })); - } else { - log('Error:', `${slug} in ${project} returned no data. Response code: ${responseCode}.`); - } - translationsRetrieved++; - }) - .on('error', (error) => { - log('Error:', `Failed to query resource ${slug} with the following error: ${error}`); - }); + let request = http.request(options, (res) => { + let xlfBuffer: string = ''; + res.on('data', (data) => xlfBuffer += data); + res.on('end', () => { + if (res.statusCode === 200) { + stream.emit('data', new File({ contents: new Buffer(xlfBuffer) })); + } else { + log('Error:', `${slug} in ${project} returned no data. Response code: ${res.statusCode}.`); + } + translationsRetrieved++; + }); + }).on('error', (err) => { + log('Error:', `Failed to query resource ${slug} with the following error: ${err}`); + }); + request.end(); }); }); } diff --git a/build/lib/typings/event-stream.d.ts b/build/lib/typings/event-stream.d.ts index 2867c3eff36..7e5ccee5e17 100644 --- a/build/lib/typings/event-stream.d.ts +++ b/build/lib/typings/event-stream.d.ts @@ -1,6 +1,7 @@ declare module "event-stream" { import { Stream } from 'stream'; import { ThroughStream } from 'through'; + import { MapStream } from 'map-stream'; function merge(streams: Stream[]): ThroughStream; function merge(...streams: Stream[]): ThroughStream; @@ -16,4 +17,5 @@ declare module "event-stream" { function mapSync(cb: (data:I) => O): ThroughStream; function map(cb: (data:I, cb:(err?:Error, data?: O)=>void) => O): ThroughStream; + function readable(asyncFunction: Function): MapStream; } \ No newline at end of file diff --git a/build/lib/typings/through.d.ts b/build/lib/typings/through.d.ts deleted file mode 100644 index 254b844fb23..00000000000 --- a/build/lib/typings/through.d.ts +++ /dev/null @@ -1,22 +0,0 @@ -// Type definitions for through -// Project: https://github.com/dominictarr/through -// Definitions by: Andrew Gaspar -// Definitions: https://github.com/borisyankov/DefinitelyTyped - -declare module "through" { - import stream = require("stream"); - - function through(write?: (data:any) => void, - end?: () => void, - opts?: { - autoDestroy: boolean; - }): through.ThroughStream; - - module through { - export interface ThroughStream extends stream.Transform { - autoDestroy: boolean; - } - } - - export = through; -} \ No newline at end of file diff --git a/build/lib/typings/vinyl.d.ts b/build/lib/typings/vinyl.d.ts deleted file mode 100644 index a85632e172b..00000000000 --- a/build/lib/typings/vinyl.d.ts +++ /dev/null @@ -1,112 +0,0 @@ -// Type definitions for vinyl 0.4.3 -// Project: https://github.com/wearefractal/vinyl -// Definitions by: vvakame , jedmao -// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped - -declare module "vinyl" { - - import fs = require("fs"); - - /** - * A virtual file format. - */ - class File { - constructor(options?: { - /** - * Default: process.cwd() - */ - cwd?: string; - /** - * Used for relative pathing. Typically where a glob starts. - */ - base?: string; - /** - * Full path to the file. - */ - path?: string; - /** - * Path history. Has no effect if options.path is passed. - */ - history?: string[]; - /** - * The result of an fs.stat call. See fs.Stats for more information. - */ - stat?: fs.Stats; - /** - * File contents. - * Type: Buffer, Stream, or null - */ - contents?: Buffer | NodeJS.ReadWriteStream; - }); - - /** - * Default: process.cwd() - */ - public cwd: string; - /** - * Used for relative pathing. Typically where a glob starts. - */ - public base: string; - /** - * Full path to the file. - */ - public path: string; - public stat: fs.Stats; - /** - * Type: Buffer|Stream|null (Default: null) - */ - public contents: Buffer | NodeJS.ReadableStream; - /** - * Returns path.relative for the file base and file path. - * Example: - * var file = new File({ - * cwd: "/", - * base: "/test/", - * path: "/test/file.js" - * }); - * console.log(file.relative); // file.js - */ - public relative: string; - - public isBuffer(): boolean; - - public isStream(): boolean; - - public isNull(): boolean; - - public isDirectory(): boolean; - - /** - * Returns a new File object with all attributes cloned. Custom attributes are deep-cloned. - */ - public clone(opts?: { contents?: boolean }): File; - - /** - * If file.contents is a Buffer, it will write it to the stream. - * If file.contents is a Stream, it will pipe it to the stream. - * If file.contents is null, it will do nothing. - */ - public pipe( - stream: T, - opts?: { - /** - * If false, the destination stream will not be ended (same as node core). - */ - end?: boolean; - }): T; - - /** - * Returns a pretty String interpretation of the File. Useful for console.log. - */ - public inspect(): string; - } - - /** - * This is required as per: - * https://github.com/Microsoft/TypeScript/issues/5073 - */ - namespace File {} - - export = File; - -} \ No newline at end of file diff --git a/package.json b/package.json index a7a83ed7133..e0093cae6d5 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,10 @@ "@types/mocha": "^2.2.39", "@types/semver": "^5.3.30", "@types/sinon": "^1.16.34", + "@types/through": "^0.0.28", "@types/winreg": "^1.2.30", + "@types/vinyl": "^2.0.0", + "@types/xml2js": "^0.0.33", "azure-storage": "^0.3.1", "clean-css": "3.4.6", "coveralls": "^2.11.11", @@ -105,7 +108,8 @@ "underscore": "^1.8.2", "vinyl": "^0.4.5", "vinyl-fs": "^2.4.3", - "vscode-nls-dev": "^2.0.1" + "vscode-nls-dev": "^2.0.1", + "xml2js": "^0.4.17" }, "repository": { "type": "git",