# pip install click
import sys
import os
import re
import xml.etree.ElementTree
from xml.sax.saxutils import escape
def validate_line_endings(path: str, content: bytes):
line = 0
for i in range(0, len(content)):
if content[i] == ord('\n'):
if i == 0 or content[i - 1] != ord('\r'):
raise RuntimeError(f'Incorrect line ending (expected CRLF) in {path}:{line}')
line += 1
def get_strings_from_file(path: str, check_line_endings: bool) -> list:
with open(path, 'rb') as fd:
content = fd.read()
if check_line_endings:
validate_line_endings(path, content)
content = xml.etree.ElementTree.fromstring(content.decode())
result = {}
for e in content.findall('./data'):
nodes = list(e.iter())
text = next(n.text for n in nodes if n.tag == 'value')
comment = next((n.text for n in nodes if n.tag == 'comment'), '')
name = e.get('name')
if name in result:
raise RuntimeError(f'error: String "{name}" is duplicated in file "{path}"')
result[name] = text, comment
return result
def cut_insert(insert: str) -> str:
index = 1
while index < len(insert) and insert[index] != '$':
index += 1
if index + 1 >= len(insert) or insert[index] != '$':
raise RuntimeError(f'Invalid insert: {insert}')
index += 1
if insert[index] in ['h', 'l']:
index += 1
return insert[0:index]
def get_inserts_in_string(string: str) -> int:
return string.replace('{{', '').count('{')
def get_file_string_inserts(strings: list) -> dict:
return {name: get_inserts_in_string(value[0]) for name, value in strings.items()}
def validate_resource(baseline: dict, path: str):
strings = get_strings_from_file(path, False)
resource = get_file_string_inserts(strings)
result = True
for string, inserts in baseline.items():
if string not in resource:
print(f'warning: string {string} found in baseline but not in {path}')
continue
if inserts != resource[string]:
print(f'error: Different inserts found for string {string}. Baseline: {inserts}, {path}: {resource[string]}')
result = False
comment = strings[string][1]
locked_strings = re.findall('{Locked="([^}]*)"}', comment, re.DOTALL)
for e in locked_strings:
if e not in strings[string][0]:
print(f'error: locked string "{e}" not found in string {string}: {strings[string][0]}')
result = False
return result
def find_argument_end(argument: str) -> int:
for i in range(len(argument)):
if not argument[i].isalnum() and argument[i] != '-' and argument[i] != '%':
# Include one extra character after the argument so that nothing gets added after the argument
# See: https://github.com/microsoft/WSL/issues/10756
return min(len(argument), i + 1)
return len(argument)
def get_locked_strings(name: str, string: str) -> tuple[list, bool]:
strings = []
def add_arguments(prefix):
for i, e in enumerate(string.split(prefix)):
if i == 0 and not e.startswith(prefix):
continue
stop_index = find_argument_end(e)
if stop_index > 1:
strings.append(prefix + e[:stop_index])
add_arguments('--')
if 'wslconfig'.lower() in name.lower():
add_arguments('/') # Edge case for wslconfig
if '.wslconfig'.lower() in string.lower():
strings.append('.wslconfig')
return strings, '{}' in string
def generate_string_comment(arguments: list, uses_insert: bool) -> str:
insert_rule = '{FixedPlaceholder="{}"}' if uses_insert else ''
return insert_rule + ''.join(f'{{Locked="{e}"}}' for e in arguments) + 'Command line arguments, file names and string inserts should not be translated'
def validate_comments(strings: dict):
result = True
comments_changes = {}
for name, (string, comment) in strings.items():
arguments, uses_insert = get_locked_strings(name, string)
if len(arguments) == 0 and not uses_insert:
continue # No command line arguments or inserts in this string
# For the sake of simplicity this logic makes the assumption that comments
# are always in the same order as of the original string
expected_comment = generate_string_comment(arguments, uses_insert)
if not expected_comment in comment:
comments_changes[name] = (comment, expected_comment)
print(f'Incorrect comment for string {name}. Expected comment: {expected_comment}')
result = False
return result, comments_changes
def fix_comments(comments: dict, path: str, strings: dict):
with open(path, 'rb') as fd:
content = fd.read()
missed = 0
for name, (comment, fixed_comment) in comments.items():
comment = comment.replace('\n', '\r\n')
matches = content.count(comment.encode())
if comment and matches == 1:
content = content.replace(comment.encode(), fixed_comment.encode())
continue
elif not comment or matches == 0:
# Try to add the comment if it doesn't exist at all
reconstructed_xml = f'''
{escape(strings[name][0])}
'''
suffix = ' '
pattern = (reconstructed_xml + suffix).replace('\n', '\r\n').encode()
if content.count(pattern) == 1:
content = content.replace(
pattern,
f'{reconstructed_xml} {fixed_comment}\n{suffix}'.replace('\n', '\r\n').encode())
continue
click.secho(f"Couldn't find unique match for comment (name={name}): {comment}. It needs to be manually replaced with: {fixed_comment}")
missed += 1
with open(path, 'wb') as fd:
fd.write(content)
click.secho(f'Updated file: {path}. {missed} comments need manual changes', fg='green' if missed == 0 else 'yellow', bold=True)
def run(resource_folder: str, baseline_language: str, fix: bool):
baseline_file = f'{resource_folder}/{baseline_language}/Resources.resw'
strings = get_strings_from_file(baseline_file, True)
baseline = get_file_string_inserts(strings)
result, comments = validate_comments(strings)
for language in os.listdir(resource_folder):
path = f'{resource_folder}/{language}/Resources.resw'
print(f'Validating inserts in {path}')
result &= validate_resource(baseline, path)
if fix and comments:
fix_comments(comments, baseline_file, strings)
sys.exit(0 if result else 1)
if __name__ == '__main__':
if len(sys.argv) == 3: # Hack to work around pip install errors in the build pipeline
run(sys.argv[1], sys.argv[2], False)
else:
import click
@click.command()
@click.argument('resource-folder', default='localization/strings')
@click.argument('baseline-language', default='en-us')
@click.option('--fix', is_flag=True)
def main(resource_folder: str, baseline_language: str, fix: bool):
run(resource_folder, baseline_language, fix)
main()